LED Light Intensity Control Using Bolt IoT & Hand Gestures(Computer Vision📷)

Introduction :

    I am Subhendu Choudhury Robotics and Automation Engineering student at THE NEOTIA UNIVERSITY. I  Build a Bolt IoT Project on light intensity control using hand gestures. In this video, you can see what I build.

Project Explanation :

  Diagram :-

              

  Code( Language python): 

           Hand_tracking_module.py :

import cv2

import mediapipe as mp

import time

import math

import numpy as np

class handDetector():

    def __init__(self, mode=False, maxHands=2, detectionCon=0.5, trackCon=0.5):

        self.mode = mode

        self.maxHands = maxHands

        self.detectionCon = detecfindHands(self, img, draw=True)tionCon

        self.trackCon = trackCon

        self.mpHands = mp.solutions.hands

        self.hands = self.mpHands.Hands(self.mode, self.maxHands,

                                        self.detectionCon, self.trackCon)

        self.mpDraw = mp.solutions.drawing_utils

        self.tipIds = [4, 8, 12, 16, 20]

    def findHands(self, img, draw=True):

        imgRGB = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)

        self.results = self.hands.process(imgRGB)

        if self.results.multi_hand_landmarks:

            for handLms in self.results.multi_hand_landmarks:

                if draw:

                    self.mpDraw.draw_landmarks(img, handLms,

                                               self.mpHands.HAND_CONNECTIONS)

        return img

    def findPosition(self, img, handNo=0, draw=True):

        xList = []

        yList = []

        bbox = []

        self.lmList = []

        if self.results.multi_hand_landmarks:

            myHand = self.results.multi_hand_landmarks[handNo]

            for id, lm in enumerate(myHand.landmark):

                # print(id, lm)

                h, w, c = img.shape

                cx, cy, cz = int(lm.x * w), int(lm.y * h), int(lm.z * c)

                xList.append(cx)

                yList.append(cy)

                #print(id, cx, cy)

                self.lmList.append([id, cx, cy])

                if draw:

                    cv2.circle(img, (cx, cy), 6, (255, 0, 255), cv2.FILLED)

            xmin, xmax = min(xList), max(xList)

            ymin, ymax = min(yList), max(yList)

            bbox = xmin, ymin, xmax, ymax

            if draw:

                cv2.rectangle(img, (xmin - 20, ymin - 20), (xmax + 20, ymax + 20),

                              (0, 255, 0), 2)

        return self.lmList, bbox

    def fingersUp(self):

        fingers = []

        # Thumb

        if self.lmList[self.tipIds[0]][1] > self.lmList[self.tipIds[0] - 1][1]:

            fingers.append(1)

        else:

            fingers.append(0)

        # Fingers

        for id in range(1, 5):

            if self.lmList[self.tipIds[id]][2] < self.lmList[self.tipIds[id] - 2][2]:

                fingers.append(1)

            else:

                fingers.append(0)

        # totalFingers = fingers.count(1)

        return fingers

    def findDistance(self, p1, p2, img, draw=True,r=15, t=3):

        x1, y1 = self.lmList[p1][1:]

        x2, y2 = self.lmList[p2][1:]

        cx, cy = (x1 + x2) // 2, (y1 + y2) // 2

        if draw:

            cv2.line(img, (x1, y1), (x2, y2), (255, 0, 255), t)

            cv2.circle(img, (x1, y1), r, (255, 0, 255), cv2.FILLED)

            cv2.circle(img, (x2, y2), r, (255, 0, 255), cv2.FILLED)

            cv2.circle(img, (cx, cy), r, (0, 0, 255), cv2.FILLED)

        length = math.hypot(x2 - x1, y2 - y1)

        return length, img, [x1, y1, x2, y2, cx, cy]

                                                                                                       *code credit to Murtaza's Workshop                    

            Light_intensity_control.py :

import cv2

import time

import numpy as np

import Hand_tracking_module as htm

import math

from boltiot import Bolt

wCam, hCam = 1280, 720 #Resolution of display size of captured image

cap = cv2.VideoCapture(0)

cap.set(3, wCam)

cap.set(4, hCam)

detector = htm.handDetector(detectionCon=0.9, maxHands=1)

intensity = 0

area = 0

colorVol = (255, 0, 0)

while True:

    success, img = cap.read()

    # Find Hand

    img = detector.findHands(img)

    lmList, bbox = detector.findPosition(img, draw=True)

    if len(lmList) != 0:

        # Filter based on size

        area = (bbox[2] - bbox[0]) * (bbox[3] - bbox[1]) // 100

        # print(area)

        if 250 < area < 1000:

            # Find Distance between index and Thumb

            length, img, lineInfo = detector.findDistance(4, 8, img)

            # print(length)

            # Convert Intensity

            Bar = np.interp(length, [50, 200], [400, 150])

            intensity = int(np.interp(length, [50, 200], [0, 255]))

            # Reduce Resolution to make it smoother

            # smoothness = 10

            # intensity = smoothness * round(intensity / smoothness)

            # Check fingers up

            fingers = detector.fingersUp()

            print(fingers)

            # If pinky is down send Intensity value to bolt server

            if not fingers[20]:

                api_key = "Put bolt IoT api key "

                device_id  = "put bolt device ID"

                mybolt = Bolt(api_key, device_id)

                response = mybolt.analogWrite('0', intensity)

                print (response)

 

                cv2.circle(img, (lineInfo[4], lineInfo[5]), 15, (0, 255, 0), cv2.FILLED)

                colorVol = (0, 255, 0)

            else:

                colorVol = (255, 0, 0)

    cv2.rectangle(img, (50, 150), (85, 400), (255, 0, 0), 3)

    cv2.rectangle(img, (50, int(Bar)), (85, 400), (255, 0, 0), cv2.FILLED)

    cv2.putText(img, f'Intensity Level: {int(intensity)}', (400, 50), cv2.FONT_HERSHEY_COMPLEX,

                1, colorVol, 3)

    cv2.imshow("Img", img)

    cv2.waitKey(1)

       . I use a Hand-Tracking module based on a media-pipe which can detect ✋hands, all fingers joints & position, fingers Up or Down, and tip distance of two-finger.

      media-pipe is a CNN(convolution Neural Network) based Computer Vision library that gives a  real-time hand-tracking solution. Using this solution I'm made a hand tracking module.

    . Using findDistance() function I measure tip distance between the index and thumb finger that gives me a variable length(range 50 - 200 experimental integer value), that's is my point to convert this length value with an Intensity value(range of 0 - 255). 

  By linearly interpolating these two ranges of value I'm getting a variable Intensity value.

            # Convert Intensity

            Bar = np.interp(length, [50, 200], [400, 150])

            intensity = int(np.interp(length, [50, 200], [0, 255]))

       . Now due to the limitation of sending API requests to Bolt Cloud, I need a trigger point that why I can control when I should send the intensity value to bolt cloud. 

 By up and down Pinky finger I use to trigger the API request(using fingersUp() function).

                                                           

            # Check fingers up

            fingers = detector.fingersUp()

            print(fingers)

            # If pinky is down send Intensity value to bolt server

            if not fingers[4]:

                api_key = "Put bolt IoT api key "

                device_id  = "put bolt device ID"

                mybolt = Bolt(api_key, device_id)

                response = mybolt.analogWrite('0', intensity)

                print (response)

 

                cv2.circle(img, (lineInfo[4], lineInfo[5]), 15, (0, 255, 0), cv2.FILLED)

                colorVol = (0, 255, 0)

            else:

                colorVol = (255, 0, 0)

 

 . When an API request sends to Bolt cloud it gives an Analog signal output to the Bolt IoT device on the defined pin. Here I define pin no. "0" as an output of an analog signal to control the intensity level of led.

                             

Conclusion & Motivation :

      My main motivation for doing this project is to showcase that we can control IoT devices using Computer Vision 📷 response and that makes a big possibility.

    Using this same method and using more optimized hardware(also some change on code) I can control an IoT-enabled robotic hand. Due to a lack of resources now I am unable to do this project but it's my future project expect.

Contact : 

Linkedin