Sign-Language-Interpreter

Runtime error

App Files Files Community

HuggingFace-SK commited on Oct 14, 2024

Commit

e60eb80

1 Parent(s): 5638c14

browser based local prediections!

Browse files

Files changed (4) hide show

README.md +1 -1
main.py +14 -0
static/style.css +65 -51
templates/browser-detect.html +306 -0

README.md CHANGED Viewed

@@ -6,7 +6,7 @@ colorTo: gray
 sdk: gradio
 sdk_version: 4.36.0
 python_version: 3.10.4
-app_file: app.py
 license: gpl-3.0
 pinned: false
 ---

 sdk: gradio
 sdk_version: 4.36.0
 python_version: 3.10.4
+app_file: main.py
 license: gpl-3.0
 pinned: false
 ---

main.py ADDED Viewed

	@@ -0,0 +1,14 @@

+from flask import Flask, render_template, Response, request, jsonify, send_from_directory
+app = Flask(__name__)
+@app.route('/')
+def index():
+    return render_template('browser-detect.html')
+@app.route('/exported')
+def send_report():
+    return send_from_directory("exported", "model.tflite")
+if (__name__ == '__main__'):
+    app.run( host='0.0.0.0', port=7860)

static/style.css CHANGED Viewed

@@ -1,79 +1,93 @@
-body {
-  --text: hsl(0 0% 15%);
-  padding: 2.5rem;
-  font-family: sans-serif;
-  color: var(--text);
-}
-body.dark-theme {
-  --text: hsl(0 0% 90%);
-  background-color: hsl(223 39% 7%);
-}
-main {
-  max-width: 80rem;
-  text-align: center;
-}
-section {
-  display: flex;
-  flex-direction: column;
-  align-items: center;
-}
-a {
-  color: var(--text);
 }
-select, input, button, .text-gen-output {
-  padding: 0.5rem 1rem;
 }
-select, img, input {
-  margin: 0.5rem auto 1rem;
 }
-form {
-  width: 25rem;
-  margin: 0 auto;
 }
-input {
-  width: 70%;
 }
-button {
-  cursor: pointer;
 }
-.text-gen-output {
-  min-height: 1.2rem;
-  margin: 1rem;
-  border: 0.5px solid grey;
 }
-#dataset button {
-  width: 6rem;
-  margin: 0.5rem;
 }
-#dataset button.hidden {
-  visibility: hidden;
 }
-table {
-  max-width: 40rem;
-  text-align: left;
-  border-collapse: collapse;
 }
-thead {
-  font-weight: bold;
 }
-td {
-  padding: 0.5rem;
 }
-td:not(thead td) {
-  border: 0.5px solid grey;
 }

+/* Copyright 2023 The MediaPipe Authors.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+     http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+video {
+  clear: both;
+  display: block;
+  transform: rotateY(180deg);
+  -webkit-transform: rotateY(180deg);
+  -moz-transform: rotateY(180deg);
 }
+section {
+  opacity: 1;
+  transition: opacity 500ms ease-in-out;
 }
+header,
+footer {
+  clear: both;
 }
+.removed {
+  display: none;
 }
+.invisible {
+  opacity: 0.2;
 }
+.note {
+  font-style: italic;
+  font-size: 130%;
 }
+.videoView,
+.detectOnClick {
+  position: relative;
+  float: left;
+  width: 48%;
+  margin: 2% 1%;
+  cursor: pointer;
 }
+.videoView p,
+.detectOnClick p {
+  position: absolute;
+  padding: 5px;
+  background-color: #007f8b;
+  color: #fff;
+  border: 1px dashed rgba(255, 255, 255, 0.7);
+  z-index: 2;
+  font-size: 12px;
+  margin: 0;
 }
+.highlighter {
+  background: rgba(0, 255, 0, 0.25);
+  border: 1px dashed #fff;
+  z-index: 1;
+  position: absolute;
 }
+.canvas {
+  z-index: 1;
+  position: absolute;
+  pointer-events: none;
 }
+.output_canvas {
+  transform: rotateY(180deg);
+  -webkit-transform: rotateY(180deg);
+  -moz-transform: rotateY(180deg);
 }
+.detectOnClick {
+  z-index: 0;
 }
+.detectOnClick img {
+  width: 100%;
 }

templates/browser-detect.html ADDED Viewed

	@@ -0,0 +1,306 @@

+<!DOCTYPE html>
+<html lang="en"><head></head>
+  <meta charset="UTF-8">
+    <link rel="apple-touch-icon" type="image/png" href="https://cpwebassets.codepen.io/assets/favicon/apple-touch-icon-5ae1a0698dcc2402e9712f7d01ed509a57814f994c660df9f7a952f3060705ee.png">
+    <meta name="apple-mobile-web-app-title" content="CodePen">
+    <link rel="shortcut icon" type="image/x-icon" href="https://cpwebassets.codepen.io/assets/favicon/favicon-aec34940fbc1a6e787974dcd360f2c6b63348d4b1f4e06c77743096d55480f33.ico">
+    <link rel="mask-icon" type="image/x-icon" href="https://cpwebassets.codepen.io/assets/favicon/logo-pin-b4b4269c16397ad2f0f7a01bcdf513a1994f4c94b8af2f191c09eb0d601762b1.svg" color="#111">
+    <script src="https://cpwebassets.codepen.io/assets/common/stopExecutionOnTimeout-2c7831bb44f98c1391d6a4ffda0e1fd302503391ca806e7fcc7b9b87197aec26.js"></script>
+  <title>CodePen - MediaPipe HandLandmarker Task for web</title>
+    <link rel="canonical" href="https://codepen.io/mediapipe-preview/pen/gOKBGPN">
+<style>
+/* Copyright 2023 The MediaPipe Authors.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+     http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+body {
+  font-family: roboto;
+  margin: 2em;
+  color: #3d3d3d;
+  --mdc-theme-primary: #007f8b;
+  --mdc-theme-on-primary: #f1f3f4;
+}
+</style>
+  <script>
+  window.console = window.console || function(t) {};
+</script>
+</head>
+<body translate="no">
+  <!-- Copyright 2023 The MediaPipe Authors.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+     http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. -->
+<script src="https://cdn.jsdelivr.net/npm/@mediapipe/drawing_utils/drawing_utils.js" crossorigin="anonymous"></script>
+<script src="https://cdn.jsdelivr.net/npm/@mediapipe/hands/hands.js" crossorigin="anonymous"></script>
+<script src="https://cdn.jsdelivr.net/npm/@tensorflow/tfjs-core"></script>
+<script src="https://cdn.jsdelivr.net/npm/@tensorflow/tfjs-backend-cpu"></script>
+<script src="https://cdn.jsdelivr.net/npm/@tensorflow/tfjs-tflite/dist/tf-tflite.min.js"></script>
+    <div id="liveView" class="videoView">
+      <img id="output_image"></img>
+      <button id="webcamButton">
+        <span>ENABLE WEBCAM</span>
+      </button>
+      <div style="position: relative;">
+        <video id="webcam" style="position: absolute" autoplay="" playsinline=""></video>
+        <canvas class="output_canvas" id="output_canvas" style="position: absolute; left: 0px; top: 0px; display:none"></canvas>
+      </div>
+    </div>
+<script type="module">
+import { HandLandmarker, FilesetResolver } from "https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.0";
+let handLandmarker = undefined;
+let runningMode = "IMAGE";
+let enableWebcamButton;
+let webcamRunning = false;
+// Before we can use HandLandmarker class we must wait for it to finish
+// loading. Machine Learning models can be large and take a moment to
+// get everything needed to run.
+const createHandLandmarker = async () => {
+    const vision = await FilesetResolver.forVisionTasks("https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.0/wasm");
+    handLandmarker = await HandLandmarker.createFromOptions(vision, {
+        baseOptions: {
+            modelAssetPath: `https://storage.googleapis.com/mediapipe-models/hand_landmarker/hand_landmarker/float16/1/hand_landmarker.task`,
+            delegate: "GPU"
+        },
+        runningMode: runningMode,
+        numHands: 1
+    });
+};
+createHandLandmarker();
+const MODEL_PATH = "/exported"
+var objectDetector = tflite.loadTFLiteModel(MODEL_PATH);
+/********************************************************************
+// Demo 2: Continuously grab image from webcam stream and detect it.
+********************************************************************/
+var global_res = 0;
+const video = document.getElementById("webcam");
+const canvasElement = document.getElementById("output_canvas");
+const canvasCtx = canvasElement.getContext("2d");
+var x_array=[]
+var y_array=[]
+// Check if webcam access is supported.
+const hasGetUserMedia = () => { var _a; return !!((_a = navigator.mediaDevices) === null || _a === void 0 ? void 0 : _a.getUserMedia); };
+// If webcam supported, add event listener to button for when user
+// wants to activate it.
+if (hasGetUserMedia()) {
+    enableWebcamButton = document.getElementById("webcamButton");
+    enableWebcamButton.addEventListener("click", enableCam);
+}
+else {
+    console.warn("getUserMedia() is not supported by your browser");
+}
+// Enable the live webcam view and start detection.
+function enableCam(event) {
+    if (!handLandmarker) {
+        console.log("Wait! objectDetector not loaded yet.");
+        return;
+    }
+    if (webcamRunning === true) {
+        webcamRunning = false;
+        enableWebcamButton.innerText = "ENABLE PREDICTIONS";
+    }
+    else {
+        webcamRunning = true;
+        enableWebcamButton.innerText = "DISABLE PREDICTIONS";
+    }
+    // getUsermedia parameters.
+    const constraints = {
+        video: true
+    };
+    // Activate the webcam stream.
+    navigator.mediaDevices.getUserMedia(constraints).then((stream) => {
+        video.srcObject = stream;
+        video.addEventListener("loadeddata", predictWebcam);
+    });
+}
+let lastVideoTime = -1;
+let results = undefined;
+console.log(video);
+async function predictWebcam() {
+    canvasElement.style.width = video.videoWidth;
+    ;
+    canvasElement.style.height = video.videoHeight;
+    canvasElement.width = video.videoWidth;
+    canvasElement.height = video.videoHeight;
+    // Now let's start detecting the stream.
+    if (runningMode === "IMAGE") {
+        runningMode = "VIDEO";
+        await handLandmarker.setOptions({ runningMode: "VIDEO" });
+    }
+    let startTimeMs = performance.now();
+    if (lastVideoTime !== video.currentTime) {
+        lastVideoTime = video.currentTime;
+        results = handLandmarker.detectForVideo(video, startTimeMs);
+    }
+    canvasCtx.save();
+    canvasCtx.clearRect(0, 0, canvasElement.width, canvasElement.height);
+    canvasCtx.drawImage(video, 0, 0, canvasElement.width, canvasElement.height)
+    if (results.landmarks) {
+        annotateImage()
+        //detectSign()
+    }
+    canvasCtx.restore();
+    // Call this function again to keep predicting when the browser is ready.
+    if (webcamRunning === true) {
+        window.requestAnimationFrame(predictWebcam);
+    }
+}
+function annotateImage(){
+//console.log(results.landmarks)
+if(results.landmarks[0]){
+        x_array=[]
+        y_array=[]
+        results.landmarks[0].forEach(iterate)
+        //console.log(x_array)
+        var image_height = video.videoHeight
+        var image_width= video.videoWidth
+    var min_x = Math.min(...x_array)*image_width
+    var min_y = Math.min(...y_array)*image_height
+    var max_x = Math.max(...x_array)*image_width
+    var max_y = Math.max(...y_array)*image_height
+    var sect_height = max_y-(min_y)
+    var sect_width = max_x-(min_x)
+    var center_x=(min_x+max_x)/2
+    var center_y=(min_y+max_y)/2
+    var sect_diameter=50
+    if(sect_height>sect_width){
+        sect_diameter = sect_height
+        //console.log("sect_height", sect_diameter)
+}
+    if(sect_height<sect_width){
+        sect_diameter = sect_width
+       // console.log("sect_width", sect_diameter)
+}
+    sect_diameter=sect_diameter+50
+    var sect_radius=sect_diameter/2
+    var crop_top=center_y-sect_radius
+    var crop_bottom=center_y+sect_radius
+    var crop_left=center_x-sect_radius
+    var crop_right=center_x+sect_radius
+    if(crop_top<0){
+        crop_top=0
+    }
+    if(crop_left<0){
+        crop_left=0
+    }
+    if(crop_right>image_width){
+        crop_right=image_width
+    }
+    if(crop_bottom>image_height){
+        crop_bottom=image_height
+    }
+    canvasCtx.beginPath();
+canvasCtx.rect(crop_left, crop_top, crop_right-crop_left, crop_bottom-crop_top);
+canvasCtx.stroke();
+}
+for (const landmarks of results.landmarks) {
+            drawConnectors(canvasCtx, landmarks, HAND_CONNECTIONS, {
+                color: "#00FF00",
+                lineWidth: 5
+            });
+            drawLandmarks(canvasCtx, landmarks, { color: "#FF0000", lineWidth: 2 });
+        }
+var dataurl=cropCanvas(canvasElement,crop_left,crop_top,244,244).toDataURL("image/jpeg", 2);
+document.getElementById("output_image").src=dataurl
+//# sourceURL=pen.js
+      }
+function iterate(x,y){
+  x_array.push(x.x)
+  y_array.push(x.y)
+}
+const cropCanvas = (sourceCanvas,left,top,width,height) => {
+    let destCanvas = document.createElement('canvas');
+    destCanvas.width = width;
+    destCanvas.height = height;
+    destCanvas.getContext("2d").drawImage(
+        sourceCanvas,
+        left,top,width,height,  // source rect with content to crop
+        0,0,width,height);      // newCanvas, same size as source
+        var predictionInput=tf.browser.fromPixels(destCanvas.getContext("2d").getImageData(0, 0, 224, 224))
+        predict(tf.expandDims(predictionInput,0))
+    return destCanvas;
+}
+async function predict(inputTensor){
+    //console.log("in predict")
+    var letter_list=["A","B","C","D","E","F","G","H","I","J","K","L","M","N","O","P","Q","R","S","T","U","V","W","X","Y","Z","#"]
+    objectDetector.then(function (res) {
+    var prediction = res.predict(inputTensor);
+    var outputArray = prediction.dataSync(); // Get the output as an array
+    var predictedClass = outputArray.indexOf(Math.max(...outputArray)); // Get the index
+    console.log(letter_list[predictedClass]);
+}, function (err) {
+    console.log(err);
+});
+}
+    </script>
+<script src="https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.0/wasm/vision_wasm_internal.js" crossorigin="anonymous"></script>
+</body>
+</html>