Initial commit

.gitignore

@@ -0,0 +1,2 @@
+/CrestronOpenCvSharp/bin/
+/CrestronOpenCvSharp/obj

.idea/.idea.CrestronOpenCvSharp/.idea/.gitignore

@@ -0,0 +1,13 @@
+# Default ignored files
+/shelf/
+/workspace.xml
+# Rider ignored files
+/contentModel.xml
+/projectSettingsUpdater.xml
+/modules.xml
+/.idea.CrestronOpenCvSharp.iml
+# Editor-based HTTP Client requests
+/httpRequests/
+# Datasource local storage ignored files
+/dataSources/
+/dataSources.local.xml

.idea/.idea.CrestronOpenCvSharp/.idea/encodings.xml

@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="Encoding" addBOMForNewFiles="with BOM under Windows, with no BOM otherwise" />
+</project>

.idea/.idea.CrestronOpenCvSharp/.idea/indexLayout.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="UserContentModel">
+    <attachedFolders />
+    <explicitIncludes />
+    <explicitExcludes />
+  </component>
+</project>

CrestronOpenCvSharp.sln

@@ -0,0 +1,16 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CrestronOpenCvSharp", "CrestronOpenCvSharp\CrestronOpenCvSharp.csproj", "{82AA53EC-314A-4435-9C78-F90441031C22}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|Any CPU = Debug|Any CPU
+		Release|Any CPU = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{82AA53EC-314A-4435-9C78-F90441031C22}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{82AA53EC-314A-4435-9C78-F90441031C22}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{82AA53EC-314A-4435-9C78-F90441031C22}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{82AA53EC-314A-4435-9C78-F90441031C22}.Release|Any CPU.Build.0 = Release|Any CPU
+	EndGlobalSection
+EndGlobal

CrestronOpenCvSharp/Capture/FacialRecognition.cs

@@ -0,0 +1,116 @@
+using FaceAiSharp;
+using FaceAiSharp.Extensions;
+using SixLabors.ImageSharp;
+using SixLabors.ImageSharp.PixelFormats;
+
+namespace CrestronOpenCvSharp.Capture;
+
+public class FacialRecognition
+{
+    private readonly IFaceDetectorWithLandmarks _detector;
+    private readonly IFaceEmbeddingsGenerator _recognizer;
+    private readonly string? _baseDirectory;
+
+    private Image<Rgb24>? _image;
+    private float[]? _referenceEmbeddings;
+
+    private readonly Dictionary<string, string> _faceImagesDict;
+    private string? FaceImagePath { get; set; }
+    
+    public FacialRecognition(string? baseDirectory)
+    {
+        _baseDirectory = baseDirectory;
+        _detector = FaceAiSharpBundleFactory.CreateFaceDetectorWithLandmarks();
+        _recognizer = FaceAiSharpBundleFactory.CreateFaceEmbeddingsGenerator();
+
+        if (_baseDirectory != null) 
+            FaceImagePath = Path.Combine(_baseDirectory, "aligned.png");
+
+        // Let's load the default stuff in this dictionary
+        _faceImagesDict = new Dictionary<string, string>
+        {
+            { "Yuri Staal", "https://ise2025.local.staal.one/VirtualControl/MA/Rooms/MYFIRSTAI/Html/yuri.jpg" },
+            { "Toine C. Leerentveld", "https://ise2025.local.staal.one/VirtualControl/MA/Rooms/MYFIRSTAI/Html/toine.jpg" },
+            { "Oliver Hall", "https://ise2025.local.staal.one/VirtualControl/MA/Rooms/MYFIRSTAI/Html/oliver.jpg" }
+        };
+    }
+
+    public bool CheckForFace(string imageFilePath)
+    {
+        
+        try
+        {
+            // Load the photo
+            var photo = File.ReadAllBytes(imageFilePath);
+            // Convert it
+            _image = Image.Load<Rgb24>(photo);
+            // Detect faces in this photo
+            var faces = _detector.DetectFaces(_image);
+
+            if (faces.Count != 0)
+            {
+                _recognizer.AlignFaceUsingLandmarks(_image, faces.First().Landmarks!);
+                _referenceEmbeddings = _recognizer.GenerateEmbedding(_image);
+                _image.Save(FaceImagePath!);
+                Console.WriteLine("Aligned faces!");
+            }
+            else
+            {
+                Console.WriteLine("No faces were found!");
+            }
+
+            // Return true or false
+            return faces.Any();
+        }
+        catch (Exception e)
+        {
+            Console.WriteLine($"Exception detecting faces: {e.Message}");
+            throw;
+        }
+    }
+
+    public async Task<string?> CompareFaces()
+    {
+        foreach (var (name, value) in _faceImagesDict)
+        {
+            var faceImage = await LoadImageAsync(value);
+            var detectedFace = _detector.DetectFaces(faceImage).FirstOrDefault();
+
+            // Generate embedding for the detected face
+            _recognizer.AlignFaceUsingLandmarks(faceImage, detectedFace.Landmarks!);
+            var faceEmbedding = _recognizer.GenerateEmbedding(faceImage);
+
+            // Compare embeddings
+            var similarity = _referenceEmbeddings?.Dot(faceEmbedding);
+            Console.WriteLine($"Similarity with {name}: {similarity}");
+            if (similarity >= 0.42)
+            {
+                //Console.WriteLine("Assessment: Both pictures show the same person.");
+                return name;
+            }
+        }
+        return null;
+    }
+
+    public void AddPersonToDatabase(string name)
+    {
+        var shortName = name.Replace(" ", "");
+        // Copy the aligned image to a new image
+        if (_baseDirectory != null)
+        {
+            var newFile = Path.Combine(_baseDirectory, $"{shortName}.jpg");
+            Console.WriteLine($"Saved new image to {newFile}");
+            File.Copy(FaceImagePath!, newFile, overwrite: true);
+        }
+
+        _faceImagesDict.Add(name, $"https://ise2025.local.staal.one/VirtualControl/MA/Rooms/MYFIRSTAI/Html/{shortName}.jpg");
+        Console.WriteLine($"Added new image to dictionary");
+    }
+    
+    private async Task<Image<Rgb24>> LoadImageAsync(string path)
+    {
+        using var hc = new HttpClient();
+        var imageBytes = await hc.GetByteArrayAsync(path);
+        return Image.Load<Rgb24>(imageBytes);
+    }
+}

CrestronOpenCvSharp/Capture/MjpegCapture.cs

@@ -0,0 +1,129 @@
+using RandomNameGeneratorLibrary;
+
+namespace CrestronOpenCvSharp.Capture;
+
+public class MjpegCapture
+{
+    private readonly HttpClient _client = new HttpClient();
+    private readonly string? _mjpegUrl;
+    private readonly string? _directory;
+    private readonly FacialRecognition? _facialRecognition;
+    private Timer? _timer;
+    private readonly PersonNameGenerator _personNameGenerator;
+
+    public bool CaptureRunning { get; private set; }
+    
+    public MjpegCapture(string? url, string? directory, FacialRecognition? facialRecognition)
+    {
+        _mjpegUrl = url;
+        _directory = Path.Combine(directory!, "captures");
+        _facialRecognition = facialRecognition;
+        
+        _personNameGenerator = new PersonNameGenerator();
+    }
+
+    public void StartCapture(int timeout)
+    {
+        if (string.IsNullOrEmpty(_directory))
+        {
+            Console.WriteLine("Directory variable was null or empty.");
+            CaptureRunning = false;
+            return;
+        }
+        CheckIfDirectoryExists(_directory);
+        // Only create if it's null
+        _timer = new Timer(CaptureImage, null, 0, timeout);
+        CaptureRunning = true;
+    }
+
+    public void StopCapture()
+    {
+        if (_timer is not null)
+        {
+            // Stop the timer
+            _timer.Change(Timeout.Infinite, Timeout.Infinite);
+            // Dispose it
+            _timer.Dispose();
+        }
+        CaptureRunning = false;
+    }
+
+    private void CheckIfDirectoryExists(string directory)
+    {
+        Console.WriteLine($"Checking if directory {directory} exists.");
+        if (Directory.Exists(directory))
+        {
+            // Remove the directory and all its contents
+            Directory.Delete(directory, true);
+            Console.WriteLine($"Directory '{directory}' and all its contents have been removed.");
+            Directory.CreateDirectory(directory);
+            Console.WriteLine($"Directory '{directory}' has been created.");
+        }
+        else
+        {
+            // Create the directory
+            Directory.CreateDirectory(directory);
+            Console.WriteLine($"Directory '{directory}' has been created.");
+        }
+    }
+    
+    private async void CaptureImage(object? state)
+    {
+        try
+        {
+            CaptureRunning = true;
+            var fileName = Path.Combine(_directory!, $"frame_{DateTime.Now:yyyyMMdd_HHmmss}.jpg");
+            try
+            {
+                HttpResponseMessage response = await _client.GetAsync(_mjpegUrl, HttpCompletionOption.ResponseHeadersRead);
+                response.EnsureSuccessStatusCode();
+
+                await using (var stream = await response.Content.ReadAsStreamAsync())
+                await using (var fileStream = new FileStream(fileName, FileMode.Create, FileAccess.Write, FileShare.None))
+                {
+                    await stream.CopyToAsync(fileStream);
+                }
+
+                Console.WriteLine($"Saved frame to {fileName}");
+                // TODO Send to image preview window on touchpanel
+                
+                if (_facialRecognition!.CheckForFace(fileName))
+                {
+                    StopCapture();
+                    Console.WriteLine("Face detected, stopping capture");    
+                    // TODO Send image to preview window on touchpanel
+                    
+                    //Path.Combine(_baseDirectory, "aligned.png"));
+                    
+                    Console.WriteLine("Comparing captured face against database");
+                    var result  = await _facialRecognition.CompareFaces();
+                    if (result is not null)
+                    {
+                        Console.WriteLine($"We have found a match! The person in front of the camera is: {result}");
+                    }
+                    else
+                    {
+                        Console.WriteLine("No match was found in our database, let's add this person!");
+                        var randomFullName = _personNameGenerator.GenerateRandomFirstAndLastName();
+                        _facialRecognition.AddPersonToDatabase(randomFullName);
+                        Thread.Sleep(2000);
+                        Console.WriteLine("Person added to database, restarting capture");
+                        StartCapture(2000);
+                    }
+                }
+                else
+                {
+                    Console.WriteLine("No face detected, carrying on!");
+                }
+            }
+            catch (Exception ex)
+            {
+                Console.WriteLine($"Error capturing image: {ex.Message}");
+            }
+        }
+        catch (Exception e)
+        {
+            Console.WriteLine($"Error capturing image: {e}");
+        }
+    }
+}

CrestronOpenCvSharp/ControlSystem.cs

@@ -0,0 +1,48 @@
+using Crestron.SimplSharp;
+using Crestron.SimplSharpPro;
+using CrestronOpenCvSharp.Capture;
+using Directory = Crestron.SimplSharp.CrestronIO.Directory;
+using Path = Crestron.SimplSharp.CrestronIO.Path;
+
+namespace CrestronOpenCvSharp;
+
+public class ControlSystem : CrestronControlSystem
+{
+    private FacialRecognition? _facialRecognition;
+    private MjpegCapture? _capture;
+    private UiHandler? _uiHandler;
+    
+    // Default snapshot URL of a Bosch camera
+    private const string Url = "http://192.168.1.221/snap.jpg";
+    private readonly string? _directory;
+    
+    public ControlSystem() : base()
+    {
+        try
+        {
+            Crestron.SimplSharpPro.CrestronThread.Thread.MaxNumberOfUserThreads = 20;
+            _directory = Path.Combine(Directory.GetApplicationRootDirectory(), $"html");
+            Console.WriteLine($"Home directory = {_directory}");
+        }
+        catch (Exception e)
+        {
+            ErrorLog.Error("Error in the constructor: {0}", e.Message);
+        }
+    }
+    
+    public override void InitializeSystem()
+    {
+        try
+        {
+            _facialRecognition = new FacialRecognition(_directory);
+            _capture = new MjpegCapture(Url, _directory, _facialRecognition);
+            _uiHandler = new UiHandler(_capture);
+            
+            _capture.StartCapture(5000);
+        }
+        catch (Exception e)
+        {
+            ErrorLog.Error("Error in InitializeSystem: {0}", e.Message);
+        }
+    }
+}

CrestronOpenCvSharp/CrestronOpenCvSharp.csproj

@@ -0,0 +1,37 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+    <PropertyGroup>
+        <TargetFramework>net8.0</TargetFramework>
+        <ImplicitUsings>enable</ImplicitUsings>
+        <Nullable>enable</Nullable>
+    </PropertyGroup>
+
+    <ItemGroup>
+      <PackageReference Include="Crestron.SimplSharp.SDK.Program" Version="2.21.90" />
+      <PackageReference Include="FaceAiSharp" Version="0.5.23" />
+      <PackageReference Include="FaceAiSharp.Bundle" Version="0.5.23" />
+      <PackageReference Include="Microsoft.ML.OnnxRuntime" Version="1.20.1" />
+      <PackageReference Include="RandomNameGeneratorLibrary" Version="1.2.2" />
+      <PackageReference Include="SixLabors.ImageSharp" Version="3.1.6" />
+    </ItemGroup>
+
+    <ItemGroup>
+      <Content Include="bin\Debug\net8.0\runtimes\linux-x64\native\libOpenCvSharpExtern.so" />
+    </ItemGroup>
+
+    <ItemGroup>
+      <Folder Include="bin\Debug\net8.0\runtimes\linux-x64\native\" />
+    </ItemGroup>
+
+    <ItemGroup>
+      <None Remove="libOpenCvSharpExtern.so" />
+    </ItemGroup>
+
+    <ItemGroup>
+      <None Remove="libonnxruntime.so" />
+      <EmbeddedResource Include="libonnxruntime.so">
+        <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+      </EmbeddedResource>
+    </ItemGroup>
+
+</Project>

CrestronOpenCvSharp/UiHandler.cs

@@ -0,0 +1,13 @@
+using CrestronOpenCvSharp.Capture;
+
+namespace CrestronOpenCvSharp;
+
+public class UiHandler
+{
+    private MjpegCapture _capture;
+    
+    public UiHandler(MjpegCapture capture)
+    {
+        _capture = capture;
+    }
+}

global.json

@@ -0,0 +1,7 @@
+{
+  "sdk": {
+    "version": "8.0.0",
+    "rollForward": "latestMinor",
+    "allowPrerelease": false
+  }
+}