Enhance Samples (#834)

* update command line parser

* clean up mqtt5 x509 sample and readme

* fix warning as error on windows

* update pubsub sample and readme

* add mqtt5 aws websocket

* update CI

* add pkcs11 sample

* add custom auth signed ample

* add unsigned custom auth

* update sample script

* wip update sample folder structure

* update sample folder structure

* wip: remove command utils

* wip : update build samples

* fix commandline

* fix cmake for commands

* fix typo

* build sample for device_defender

* fix uuid

* fix compile error and sample cmake

* remove command utils

* fix shadow update

* basic report

* update error print for DDTestRun, remove ca_file

* fix dd test run path & mqtt3 ci

* add print to argument lists

* fix script for new argument parasing

* remove prints and adjust commands

* remove job id

* remove linux smoke test

* update cr changes

* add back mqtt3 service tests

* remove jobid

Author: xiazhvera

.builder/actions/build_samples.py

@@ -19,23 +19,26 @@ def run(self, env):
 
         steps = []
         samples = [
-            'samples/commands/commands-sandbox',
+            'samples/service_clients/commands/commands-sandbox',
             'samples/greengrass/basic_discovery',
             'samples/greengrass/ipc',
-            'samples/fleet_provisioning/provision-basic',
-            'samples/fleet_provisioning/provision-csr',
-            'samples/jobs/jobs-sandbox',
-            'samples/mqtt5/mqtt5_pubsub',
-            'samples/secure_tunneling/secure_tunnel',
-            'samples/secure_tunneling/tunnel_notification',
-            'samples/shadow/shadow-sandbox',
+            'samples/service_clients/fleet_provisioning/provision-basic',
+            'samples/service_clients/fleet_provisioning/provision-csr',
+            'samples/service_clients/jobs/jobs-sandbox',
+            'samples/mqtt/mqtt5_x509',
+            'samples/mqtt/mqtt5_aws_websocket',
+            'samples/mqtt/mqtt5_custom_auth_signed',
+            'samples/mqtt/mqtt5_custom_auth_unsigned',
+            'samples/mqtt/mqtt5_pkcs11',
+            'samples/others/secure_tunneling/secure_tunnel',
+            'samples/others/secure_tunneling/tunnel_notification',
+            'samples/service_clients/shadow/shadow-sandbox',
         ]
 
         defender_samples = []
         # Linux only builds
         if sys.platform == "linux" or sys.platform == "linux2":
-            defender_samples.append('samples/device_defender/basic_report')
-            defender_samples.append('samples/device_defender/mqtt5_basic_report')
+            defender_samples.append('samples/others/device_defender/mqtt5_basic_report')
 
         servicetests = [
             'servicetests/tests/JobsExecution/',

.github/workflows/ci.yml

@@ -240,7 +240,7 @@ jobs:
         run: |
           cd ${{ env.CI_FOLDER }}
           echo "Starting to run AppVerifier with mqtt5 pub-sub sample"
-          python ${{ env.CI_UTILS_FOLDER }}/appverifier_launch_sample.py --sample_file ".\aws-iot-device-sdk-cpp-v2\build\samples\mqtt5\mqtt5_pubsub\RelWithDebInfo\mqtt5_pubsub.exe" --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/mqtt5/us/mqtt5_thing/cert' --sample_secret_private_key 'ci/mqtt5/us/mqtt5_thing/key'
+          python ${{ env.CI_UTILS_FOLDER }}/appverifier_launch_sample.py --sample_file ".\aws-iot-device-sdk-cpp-v2\build\samples\mqtt\mqtt5_x509\RelWithDebInfo\mqtt5_x509.exe" --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/mqtt5/us/mqtt5_thing/cert' --sample_secret_private_key 'ci/mqtt5/us/mqtt5_thing/key'
 
   windows-shared-lib:
     runs-on: windows-latest

codebuild/linux-integration-tests.yml

@@ -15,7 +15,7 @@ phases:
     commands:
       - echo Build started on `date`
       # Building of dependencies happens in setup-linux
-      - $CODEBUILD_SRC_DIR/codebuild/samples/setup-linux.sh
+      - $CODEBUILD_SRC_DIR/codebuild/setup-linux.sh
 
       # Run the integration tests
       - $CODEBUILD_SRC_DIR/codebuild/integration-tests.sh

codebuild/samples/linux-smoke-tests.yml

@@ -43,10 +43,6 @@ def delete_thing_with_certi(thingName, certiId, certiArn):
 thing_arn = None
 client_made_thing = False
 client_made_policy = False
-use_mqtt5 = False
-if len(sys.argv) > 1:
-    use_mqtt5 = (sys.argv[1] == "mqtt5")
-    print("Run Device Defender with Mqtt5 Client")
 
 ##############################################
 # create a test thing
@@ -227,30 +223,21 @@ def delete_thing_with_certi(thingName, certiId, certiArn):
 
     print("[Device Defender]Info: Running sample (this should take ~60 seconds).")
 
-    if use_mqtt5:
-        # Run the sample:
-        exe_path = "build/samples/device_defender/mqtt5_basic_report/"
-        # If running locally, comment out the line above and uncomment the line below:
-        #exe_path = "samples/device_defender/basic_report/build/"
+    # Run the sample:
+    exe_path = "build/samples/others/device_defender/mqtt5_basic_report/"
+    # If running locally, comment out the line above and uncomment the line below:
+    #exe_path = "samples/others/device_defender/mqtt5_basic_report/build/"
 
-        # Windows has a different build folder structure, but this ONLY runs on Linux currently so we do not need to worry about it
-        exe_path = os.path.join(exe_path, "mqtt5-basic-report")
-    else:
-        # Run the sample:
-        exe_path = "build/samples/device_defender/basic_report/"
-        # If running locally, comment out the line above and uncomment the line below:
-        #exe_path = "samples/device_defender/basic_report/build/"
-
-        # Windows has a different build folder structure, but this ONLY runs on Linux currently so we do not need to worry about it
-        exe_path = os.path.join(exe_path, "basic-report")
+    # Windows has a different build folder structure, but this ONLY runs on Linux currently so we do not need to worry about it
+    exe_path = os.path.join(exe_path, "mqtt5-basic-report")
 
     print("[Device Defender]Info: Start to run: " + exe_path)
     # The Device Defender sample will take ~1 minute to run even if successful
     # (since samples are sent every minute)
     arguments = [exe_path, "--endpoint", endpoint_response, "--cert",
                  certificate_path, "--key", key_path, "--thing_name", thing_name, "--count", "2"]
     result = subprocess.run(arguments, timeout=60*2, check=True)
-    print("[Device Defender]Info: Sample finished running.")
+    print(f"[Device Defender]Info: Sample finished running, with result {result.returncode}")
 
     # There does not appear to be any way to get the metrics from the device - so we'll assume that if it didn't return -1, then it worked
 
@@ -272,7 +259,7 @@ def delete_thing_with_certi(thingName, certiId, certiArn):
     if client_made_policy:
         client.delete_policy(policyName=thing_name + "_policy")
 
-    print("[Device Defender]Error: Failed to test: Basic Report")
+    print(f"[Device Defender]Error: Failed to test: Basic Report {e}")
     exit(-1)
 
 print("[Device Defender]Info: Basic Report sample test passed")

codebuild/samples/setup-linux.sh renamed to codebuild/setup-linux.sh

@@ -20,9 +20,14 @@ If you are just getting started make sure you [install this sdk](https://github.
 ### How do I enable logging?
 
 ``` c++
-ApiHandle apiHandle;
-apiHandle.InitializeLogging(Aws::Crt::LogLevel::Error, stderr);
+#include <aws/crt/Api.h>
+
+Aws::Crt::ApiHandle apiHandle;
+apiHandle.InitializeLogging(Aws::Crt::LogLevel::Debug, stderr);
 ```
+
+**LogLevel**: LogLevel has the following options: `Trace`, `Debug`, `Info`, `Warn`, `Error`, `Fatal`, or `None`. Defaults to `Warn`.
+
 You can also enable [CloudWatch logging](https://docs.aws.amazon.com/iot/latest/developerguide/cloud-watch-logs.html) for IoT which will provide you with additional information that is not available on the client side sdk.
 
 ### I keep getting AWS_ERROR_MQTT_UNEXPECTED_HANGUP
@@ -99,7 +104,6 @@ Here is an example launch.json file to run the pubsub sample
             "program": "${workspaceFolder}/samples/pub_sub/basic_pub_sub/build/basic-pub-sub",
             "args": [
                 "--endpoint", "<account-number>-ats.iot.<region>.amazonaws.com",
-                "--ca_file", "<path to root-CA>",
                 "--cert", "<path to cert>",
                 "--key", "<path to key>",
                 "--client-id", "test-client"
@@ -116,7 +120,6 @@ Here is an example launch.json file to run the pubsub sample
     * Root CA Certificates
         * Download the root CA certificate file that corresponds to the type of data endpoint and cipher suite you're using (You most likely want Amazon Root CA 1)
         * Generated and provided by Amazon. You can download it [here](https://www.amazontrust.com/repository/) or download it when getting the other certificates from the AWS console
-        * When using samples it can look like this: `--ca_file root-CA.crt`
     * Device certificate
         * Intermediate device certificate that is used to generate the key below
         * When using samples it can look like this: `--cert abcde12345-certificate.pem.crt`

devicedefender/script/DDTestRun.py

@@ -1369,8 +1369,8 @@ samples.
 
 It's always helpful to look at a working example to see how new functionality works,
 to be able to tweak different options, to compare with existing code.
-For that reason, we implemented a [Publish/Subscribe example](https://github.com/aws/aws-iot-device-sdk-cpp-v2/tree/main/samples/mqtt5/mqtt5_pubsub)
-([source code](https://github.com/aws/aws-iot-device-sdk-cpp-v2/blob/main/samples/mqtt5/mqtt5_pubsub/main.cpp))
+For that reason, we implemented a [X509 Publish/Subscribe example](https://github.com/aws/aws-iot-device-sdk-cpp-v2/tree/main/samples/mqtt/mqtt5_x509)
+([source code](https://github.com/aws/aws-iot-device-sdk-cpp-v2/blob/main/samples/mqtt/mqtt5_x509/main.cpp))
 in the v2 SDK similar to a sample provided by the v1 SDK (see a corresponding
 [readme section](https://github.com/aws/aws-iot-device-sdk-cpp/blob/master/samples/README.md) and
 [source code](https://github.com/aws/aws-iot-device-sdk-cpp/blob/master/samples/PubSub/PubSub.cpp)).
@@ -1438,9 +1438,7 @@ method in PublishPacketBuilder class.
 
 **Shared Subscriptions**\
 Shared Subscriptions allow multiple clients to share a subscription to a topic and only one client
-will receive messages published to that topic using a random distribution.\
-For more information, see a [shared subscription sample](https://github.com/aws/aws-iot-device-sdk-cpp-v2/blob/main/samples/mqtt5/mqtt5_shared_subscription/README.md)
-in the v2 SDK.
+will receive messages published to that topic using a random distribution.
 
 > [!NOTE]
 > AWS Iot Core supports Shared Subscriptions for both MQTT3 and MQTT5. For more information, see

documents/FAQ.md

@@ -249,9 +249,9 @@ secureTunnel->SendMessage(message);
 
 # Secure Tunnel Best Practices
 * You MUST NOT perform blocking operations on any callback, or you will cause a deadlock.
-* If you do not provide a Client Token during creation of the Secure Tunnel, one will be automatically generated for you to use in reconnections. This token is not saved outside of the current Secure Tunnel Client. If the Client is destroyed, the original access tokens must be rotated to connect to the secure tunnel again. Information on rotating tokens can be found here: https://docs.aws.amazon.com/iot/latest/developerguide/iot-secure-tunneling-troubleshooting.html
+* If you do not provide a Client Token during creation of the Secure Tunnel, one will be automatically generated for you to use in reconnection. This token is not saved outside of the current Secure Tunnel Client. If the Client is destroyed, the original access tokens must be rotated to connect to the secure tunnel again. Information on rotating tokens can be found here: https://docs.aws.amazon.com/iot/latest/developerguide/iot-secure-tunneling-troubleshooting.html
 * Client tokens MUST be unique. You cannot for example, pair a Client Token with an Access Token on one secure tunnel, and then use the same Client Token with a different Access Token on a separate secure tunnel. The Secure Tunnel Service will not allow a Client Token to be paired with more than one Access Token.
 * A Secure Tunnel Client that has called `Start()` will continue to attempt to connect the Secure Tunnel Service until `Stop()` is called, even if the Secure Tunnel it is trying to connect with has been closed. You MUST call `Stop()` to cease future connection attempts.
-* The [onStreamStarted](#onstreamstarted) and [onConnectionStarted](#onconnectionstarted) callbacks should be set to detect and store the service id and/or connection id of streams started by a source device for use with messages. The [Secure Tunnel sample](../samples/secure_tunneling/secure_tunnel/README.md) provides an basic example of how this can be done.
+* The [onStreamStarted](#onstreamstarted) and [onConnectionStarted](#onconnectionstarted) callbacks should be set to detect and store the service id and/or connection id of streams started by a source device for use with messages. The [Secure Tunnel sample](../samples/others/secure_tunneling/secure_tunnel/README.md) provides an basic example of how this can be done.
 * Outgoing messages MUST be assigned a service id and/or a connection id if the established stream contains a service id or a connection id or the message will be rejected. e.g. If a stream is started using service id "ssh" and connection id (1), a message sent in response must also include the service id "ssh" and connection id (1) or it will not find an active stream to send it on. Refer to the [Send Message](#send-message) code block for instruction on adding a service id and/or connection id to your message.
 

documents/MIGRATION_GUIDE.md

@@ -2,15 +2,18 @@ cmake_minimum_required(VERSION 3.9...3.31)
 
 project(aws-iot-device-sdk-cpp-v2-samples)
 
-add_subdirectory(device_defender/basic_report)
-add_subdirectory(device_defender/mqtt5_basic_report)
-add_subdirectory(fleet_provisioning/provision-basic)
-add_subdirectory(fleet_provisioning/provision-csr)
+add_subdirectory(others/device_defender/mqtt5_basic_report)
+add_subdirectory(service_clients/fleet_provisioning/provision-basic)
+add_subdirectory(service_clients/fleet_provisioning/provision-csr)
 add_subdirectory(greengrass/ipc)
 add_subdirectory(greengrass/basic_discovery)
-add_subdirectory(jobs/jobs-sandbox)
-add_subdirectory(mqtt5/mqtt5_pubsub)
-add_subdirectory(secure_tunneling/secure_tunnel)
-add_subdirectory(secure_tunneling/tunnel_notification)
-add_subdirectory(shadow/shadow_sync)
-add_subdirectory(commands/commands-sandbox)
+add_subdirectory(service_clients/jobs/jobs-sandbox)
+add_subdirectory(mqtt/mqtt5_x509)
+add_subdirectory(mqtt/mqtt5_aws_websocket)
+add_subdirectory(mqtt/mqtt5_custom_auth_signed)
+add_subdirectory(mqtt/mqtt5_custom_auth_unsigned)
+add_subdirectory(mqtt/mqtt5_pkcs11)
+add_subdirectory(others/secure_tunneling/secure_tunnel)
+add_subdirectory(others/secure_tunneling/tunnel_notification)
+add_subdirectory(service_clients/shadow/shadow-sandbox)
+add_subdirectory(service_clients/commands/commands-sandbox)