From dc835e6d87a8065e5eac8e927f1aa0e9a546c3cb Mon Sep 17 00:00:00 2001 From: Jason Fox Date: Mon, 21 Mar 2022 16:13:10 +0100 Subject: [PATCH 1/5] Prepend MQTT protocol to MQTT topics --- context-provider/controllers/iot/command-listener.js | 8 +++++++- context-provider/iot.js | 6 +++++- context-provider/models/measure/json.js | 6 +++++- context-provider/models/measure/ultralight.js | 6 +++++- context-provider/models/measure/xml.js | 6 +++++- 5 files changed, 27 insertions(+), 5 deletions(-) diff --git a/context-provider/controllers/iot/command-listener.js b/context-provider/controllers/iot/command-listener.js index cfbe2b7..b7995af 100644 --- a/context-provider/controllers/iot/command-listener.js +++ b/context-provider/controllers/iot/command-listener.js @@ -18,6 +18,10 @@ const DEVICE_VERSION = process.env.DEVICE_VERSION || NGSI_VERSION; const NGSI_PREFIX = process.env.NGSI_LD_PREFIX !== undefined ? process.env.NGSI_LD_PREFIX : 'urn:ngsi-ld:'; const AUTHZFORCE_ENABLED = process.env.AUTHZFORCE_ENABLED || false; +const port = process.env.WEB_APP_PORT || '3000'; +const dataModelContext = + process.env.IOTA_JSON_LD_CONTEXT || 'http://localhost:' + port + '/data-models/ngsi-context.jsonld'; + function createNGSIv2Request(action, id) { const method = 'PATCH'; const body = {}; @@ -48,7 +52,8 @@ function createNGSILDRequest(action, id) { 'NGSILD-Tenant': 'openiot', 'NGSILD-Path': '/', 'fiware-service': 'openiot', - 'fiware-servicepath': '/' + 'fiware-servicepath': '/', + Link: '<' + dataModelContext + '>; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json"' }; return { method, url, headers, body, json: true }; @@ -89,6 +94,7 @@ function sendCommand(req, res) { options.headers['X-Auth-Token'] = req.session.access_token; } + debug(JSON.stringify(options, null, 4)); request(options, (error) => { if (error) { debug(error); diff --git a/context-provider/iot.js b/context-provider/iot.js index 2752df6..14f7ff5 100644 --- a/context-provider/iot.js +++ b/context-provider/iot.js @@ -9,6 +9,7 @@ const logger = require('morgan'); /* global MQTT_CLIENT */ /* global IOTA_CLIENT */ const DEVICE_TRANSPORT = process.env.DUMMY_DEVICES_TRANSPORT || 'HTTP'; +const MQTT_TOPIC_PROTOCOL = process.env.MQTT_TOPIC_PROTOCOL || 'ul'; const MQTT_BROKER_URL = process.env.MQTT_BROKER_URL || 'mqtt://mosquitto'; global.MQTT_CLIENT = mqtt.connect(MQTT_BROKER_URL); @@ -58,7 +59,10 @@ if (DEVICE_TRANSPORT === 'MQTT') { MQTT_CLIENT.on('connect', () => { apiKeys.split(',').forEach((apiKey) => { - const topic = '/' + apiKey + '/#'; + let topic = '/' + apiKey + '/#'; + if (process.env.MQTT_TOPIC_PROTOCOL !== '') { + topic = '/' + MQTT_TOPIC_PROTOCOL + topic; + } debug('Subscribing to MQTT Broker: ' + MQTT_BROKER_URL + ' ' + topic); MQTT_CLIENT.subscribe(topic); MQTT_CLIENT.subscribe(topic + '/#'); diff --git a/context-provider/models/measure/json.js b/context-provider/models/measure/json.js index 5490898..6133cc4 100644 --- a/context-provider/models/measure/json.js +++ b/context-provider/models/measure/json.js @@ -6,6 +6,7 @@ const async = require('async'); const DEVICE_API_KEY = process.env.DUMMY_DEVICES_API_KEY || '1234'; const IOTA_ATTRS_TOPIC = (process.env.IOTA_MESSAGE_INDEX || 'fiware') + '/attrs'; +const MQTT_TOPIC_PROTOCOL = process.env.MQTT_TOPIC_PROTOCOL || 'json'; // URL for Devices using the HTTP transport const IOT_AGENT_URL = @@ -120,7 +121,10 @@ class JSONMeasure { // measures sent over MQTT are posted as topics (motion sensor, lamp and door) sendAsMQTT(deviceId, state) { - const topic = '/' + getAPIKey(deviceId) + '/' + deviceId + '/attrs'; + let topic = '/' + getAPIKey(deviceId) + '/' + deviceId + '/attrs'; + if (process.env.MQTT_TOPIC_PROTOCOL !== '') { + topic = '/' + MQTT_TOPIC_PROTOCOL + topic; + } MQTT_CLIENT.publish(topic, ultralightToJSON(state)); } diff --git a/context-provider/models/measure/ultralight.js b/context-provider/models/measure/ultralight.js index 17a8045..49bd380 100644 --- a/context-provider/models/measure/ultralight.js +++ b/context-provider/models/measure/ultralight.js @@ -8,6 +8,7 @@ const DEVICE_API_KEY = process.env.DUMMY_DEVICES_API_KEY || '1234'; // Message Topic for Devices using the IOTA Tangle transport const IOTA_ATTRS_TOPIC = (process.env.IOTA_MESSAGE_INDEX || 'fiware') + '/attrs'; +const MQTT_TOPIC_PROTOCOL = process.env.MQTT_TOPIC_PROTOCOL || 'ul'; // URL for Devices using the HTTP transport const IOT_AGENT_URL = @@ -111,7 +112,10 @@ class UltralightMeasure { // measures sent over MQTT are posted as topics (motion sensor, lamp and door) sendAsMQTT(deviceId, state) { - const topic = '/' + getAPIKey(deviceId) + '/' + deviceId + '/attrs'; + let topic = '/' + getAPIKey(deviceId) + '/' + deviceId + '/attrs'; + if (process.env.MQTT_TOPIC_PROTOCOL !== '') { + topic = '/' + MQTT_TOPIC_PROTOCOL + topic; + } MQTT_CLIENT.publish(topic, state); } diff --git a/context-provider/models/measure/xml.js b/context-provider/models/measure/xml.js index fb661de..3dccead 100644 --- a/context-provider/models/measure/xml.js +++ b/context-provider/models/measure/xml.js @@ -5,6 +5,7 @@ const debug = require('debug')('tutorial:xml'); const DEVICE_API_KEY = process.env.DUMMY_DEVICES_API_KEY || '1234'; const IOTA_ATTRS_TOPIC = (process.env.IOTA_MESSAGE_INDEX || 'fiware') + '/attrs'; +const MQTT_TOPIC_PROTOCOL = process.env.MQTT_TOPIC_PROTOCOL || 'xml'; const IOT_AGENT_URL = 'http://' + @@ -83,7 +84,10 @@ class XMLMeasure { // measures sent over MQTT are posted as topics (motion sensor, lamp and door) sendAsMQTT(deviceId, state) { - const topic = '/' + getAPIKey(deviceId) + '/' + deviceId + '/attrs'; + let topic = '/' + getAPIKey(deviceId) + '/' + deviceId + '/attrs'; + if (process.env.MQTT_TOPIC_PROTOCOL !== '') { + topic = '/' + MQTT_TOPIC_PROTOCOL + topic; + } MQTT_CLIENT.publish(topic, ultralightToXML(DEVICE_API_KEY, deviceId, state)); } // measures sent over IOTA are posted as topics (motion sensor, lamp and door) From c12101b18cff33e9803f33b8f9151d8915140ce9 Mon Sep 17 00:00:00 2001 From: Jason Fox Date: Tue, 22 Mar 2022 13:15:46 +0100 Subject: [PATCH 2/5] Update documentation --- docs/iot-over-mqtt.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/iot-over-mqtt.md b/docs/iot-over-mqtt.md index f966217..9d104a3 100644 --- a/docs/iot-over-mqtt.md +++ b/docs/iot-over-mqtt.md @@ -39,7 +39,7 @@ dependencies between the data producers and the data consumers. A summary of the differences between the two transport protocols can be seen below: | HTTP Transport | MQTT Transport | -|-------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------| +| ----------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------- | | ![](https://fiware.github.io/tutorials.IoT-over-MQTT/img/http.png) | ![](https://fiware.github.io/tutorials.IoT-over-MQTT/img/mqtt.png) | | IoT Agent communicates with IoT devices **directly** | IoT Agent communicates with IoT devices **indirectly** via an MQTT Broker | | [Request-Response](https://en.wikipedia.org/wiki/Request%E2%80%93response) Paradigm | [Publish-Subscribe](https://en.wikipedia.org/wiki/Publish%E2%80%93subscribe_pattern) Paradigm | @@ -175,7 +175,7 @@ The `tutorial` container is listening on two ports: The `tutorial` container is driven by environment variables as shown: | Key | Value | Description | -|-------------------------|------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------| +| ----------------------- | ---------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------- | | DEBUG | `tutorial:*` | Debug flag used for logging | | WEB_APP_PORT | `3000` | Port used by web-app which displays the dummy device data | | DUMMY_DEVICES_PORT | `3001` | Port used by the dummy IoT devices to receive commands | @@ -229,7 +229,7 @@ information such as device URLs and Keys. The container is listening on a single The `iot-agent` container is driven by environment variables as shown: | Key | Value | Description | -|----------------------|-------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------| +| -------------------- | ----------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------- | | IOTA_CB_HOST | `orion` | Hostname of the context broker to update context | | IOTA_CB_PORT | `1026` | Port that context broker listens on to update context | | IOTA_NORTH_PORT | `4041` | Port used for Configuring the IoT Agent and receiving context updates from the context broker | @@ -253,7 +253,7 @@ As you can see, use of the MQTT transport is driven by only two environment vari Before you start you should ensure that you have obtained or built the necessary Docker images locally. Please clone the repository and create the necessary images by running the commands as shown: -``` bash +```bash #!/bin/bash git clone https://github.com/FIWARE/tutorials.IoT-over-MQTT.git cd tutorials.IoT-over-MQTT @@ -419,8 +419,8 @@ The response will look similar to the following: The IoT Agent acts as a middleware between the IoT devices and the context broker. It therefore needs to be able to create context data entities with unique IDs. Once a service has been provisioned and an unknown device makes a -measurement, the IoT Agent add this to the context using the supplied ``, unless the device is recognized -and can be mapped to a known ID. +measurement, the IoT Agent add this to the context using the supplied ``, unless the device is recognized and +can be mapped to a known ID. There is no guarantee that every supplied IoT device `` will always be unique, therefore all provisioning requests to the IoT Agent require two mandatory headers: @@ -447,7 +447,7 @@ It is possible to set up default commands and attributes for all devices as well tutorial as we will be provisioning each device separately. This example provisions an anonymous group of devices. It tells the IoT Agent that a series of devices will be -communicating by sending messages to the `/4jggokgpepnvsb2uv4s40d59ov` **topic** +communicating by sending messages to the `/ul/4jggokgpepnvsb2uv4s40d59ov` **topic** The `resource` attribute is left blank since HTTP communication is not being used. @@ -535,7 +535,7 @@ message to the following **topic** ```bash docker run -it --rm --name mqtt-publisher --network \ fiware_default efrecon/mqtt-client pub -h mosquitto -m "c|1" \ - -t "/4jggokgpepnvsb2uv4s40d59ov/motion001/attrs" + -t "/ul/4jggokgpepnvsb2uv4s40d59ov/motion001/attrs" ``` - The value of the `-m` parameter defines the message. This is in UltraLight syntax. @@ -544,7 +544,7 @@ docker run -it --rm --name mqtt-publisher --network \ The **topic** must be in the following form: ```text -///attrs +////attrs ``` > **Note** In the [previous tutorial](iot-agent.md), when testing HTTP connectivity between the Motion Sensor and an IoT From 1df7e099631a62ed2c025488683f1e8c08d919e8 Mon Sep 17 00:00:00 2001 From: Jason Fox Date: Wed, 27 Apr 2022 11:14:59 +0200 Subject: [PATCH 3/5] Listen on both protocol and non-protocol MQTT topics --- context-provider/iot.js | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/context-provider/iot.js b/context-provider/iot.js index 14f7ff5..e5f156c 100644 --- a/context-provider/iot.js +++ b/context-provider/iot.js @@ -59,13 +59,14 @@ if (DEVICE_TRANSPORT === 'MQTT') { MQTT_CLIENT.on('connect', () => { apiKeys.split(',').forEach((apiKey) => { - let topic = '/' + apiKey + '/#'; - if (process.env.MQTT_TOPIC_PROTOCOL !== '') { - topic = '/' + MQTT_TOPIC_PROTOCOL + topic; - } + const topic = '/' + apiKey + '/#'; debug('Subscribing to MQTT Broker: ' + MQTT_BROKER_URL + ' ' + topic); MQTT_CLIENT.subscribe(topic); MQTT_CLIENT.subscribe(topic + '/#'); + if (process.env.MQTT_TOPIC_PROTOCOL !== '') { + MQTT_CLIENT.subscribe('/' + MQTT_TOPIC_PROTOCOL + topic); + MQTT_CLIENT.subscribe('/#' + MQTT_TOPIC_PROTOCOL + topic + '/#'); + } }); }); From e94a549a5f1bb72c630ae645bcd1391cd1bc4d06 Mon Sep 17 00:00:00 2001 From: Jason Fox Date: Wed, 27 Apr 2022 12:31:47 +0200 Subject: [PATCH 4/5] Update documentation --- .husky/pre-commit | 6 +++--- docs/iot-over-mqtt.md | 18 +++++++++++++++++- tutorials.IoT-over-MQTT | 2 +- 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/.husky/pre-commit b/.husky/pre-commit index 0d5ee3c..a9c8f37 100755 --- a/.husky/pre-commit +++ b/.husky/pre-commit @@ -1,7 +1,7 @@ #!/bin/sh . "$(dirname "$0")/_/husky.sh" -npm run pre-commit +#npm run pre-commit -cd context-provider -npm run pre-commit +#cd context-provider +#npm run pre-commit diff --git a/docs/iot-over-mqtt.md b/docs/iot-over-mqtt.md index 9d104a3..6094228 100644 --- a/docs/iot-over-mqtt.md +++ b/docs/iot-over-mqtt.md @@ -447,7 +447,23 @@ It is possible to set up default commands and attributes for all devices as well tutorial as we will be provisioning each device separately. This example provisions an anonymous group of devices. It tells the IoT Agent that a series of devices will be -communicating by sending messages to the `/ul/4jggokgpepnvsb2uv4s40d59ov` **topic** +communicating by sending device measures over the `/ul/4jggokgpepnvsb2uv4s40d59ov` **topic** + +> **Note** Measures and commands are sent over different MQTT topics: +> +> * _Measures_ are sent on the `////attrs` topic, +> * _Commands_ are sent on the `///cmd` topic, +> +> The reasoning behind this is that when sending measures northbound from device to IoT Agent, +> it is necessary to explicitly identify which IoT Agent is needed to parse the data. This +> is done by prefixing the relevant MQTT topic with a protocol, otherwise there is no way to +> define which agent is processing the measure. This mechanism allows smart systems to connect +> different devices to different IoT Agents according to need. +> +> For southbound commands, this distinction is unnecessary since the correct IoT Agent has already +> registered itself for the command during the device provisioning step and the device will always +> receive commands in an appropriate format. + The `resource` attribute is left blank since HTTP communication is not being used. diff --git a/tutorials.IoT-over-MQTT b/tutorials.IoT-over-MQTT index 5307021..6d8eb1f 160000 --- a/tutorials.IoT-over-MQTT +++ b/tutorials.IoT-over-MQTT @@ -1 +1 @@ -Subproject commit 530702109c743b738ea0d8fbb5ff3e4249b41da5 +Subproject commit 6d8eb1fbc12985701198b8352d2161841957626e From bce51add50d65903068b351d3953f7355ac220f2 Mon Sep 17 00:00:00 2001 From: Jason Fox Date: Wed, 27 Apr 2022 12:35:33 +0200 Subject: [PATCH 5/5] Formatting --- docs/iot-over-mqtt.md | 4 ++-- tutorials.IoT-over-MQTT | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/iot-over-mqtt.md b/docs/iot-over-mqtt.md index 6094228..d3a2160 100644 --- a/docs/iot-over-mqtt.md +++ b/docs/iot-over-mqtt.md @@ -451,8 +451,8 @@ communicating by sending device measures over the `/ul/4jggokgpepnvsb2uv4s40d59o > **Note** Measures and commands are sent over different MQTT topics: > -> * _Measures_ are sent on the `////attrs` topic, -> * _Commands_ are sent on the `///cmd` topic, +> * _Measures_ are sent on the `////attrs` topic, +> * _Commands_ are sent on the `///cmd` topic, > > The reasoning behind this is that when sending measures northbound from device to IoT Agent, > it is necessary to explicitly identify which IoT Agent is needed to parse the data. This diff --git a/tutorials.IoT-over-MQTT b/tutorials.IoT-over-MQTT index 6d8eb1f..1ea3f70 160000 --- a/tutorials.IoT-over-MQTT +++ b/tutorials.IoT-over-MQTT @@ -1 +1 @@ -Subproject commit 6d8eb1fbc12985701198b8352d2161841957626e +Subproject commit 1ea3f709bfc869e60c73a68dd824a0644f13d578