commit 1f7b9770c96d3c9b82c15cd422af2b629a6cc25b Author: yuri.staal Date: Sun Feb 23 20:49:56 2025 +0100 Initial commit diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000..494b86a Binary files /dev/null and b/.DS_Store differ diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..7ffb7bf --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +/CrestronOpenCvSharp/bin/ +/CrestronOpenCvSharp/obj diff --git a/.idea/.idea.CrestronOpenCvSharp/.idea/.gitignore b/.idea/.idea.CrestronOpenCvSharp/.idea/.gitignore new file mode 100644 index 0000000..f471655 --- /dev/null +++ b/.idea/.idea.CrestronOpenCvSharp/.idea/.gitignore @@ -0,0 +1,13 @@ +# Default ignored files +/shelf/ +/workspace.xml +# Rider ignored files +/contentModel.xml +/projectSettingsUpdater.xml +/modules.xml +/.idea.CrestronOpenCvSharp.iml +# Editor-based HTTP Client requests +/httpRequests/ +# Datasource local storage ignored files +/dataSources/ +/dataSources.local.xml diff --git a/.idea/.idea.CrestronOpenCvSharp/.idea/encodings.xml b/.idea/.idea.CrestronOpenCvSharp/.idea/encodings.xml new file mode 100644 index 0000000..df87cf9 --- /dev/null +++ b/.idea/.idea.CrestronOpenCvSharp/.idea/encodings.xml @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/.idea/.idea.CrestronOpenCvSharp/.idea/indexLayout.xml b/.idea/.idea.CrestronOpenCvSharp/.idea/indexLayout.xml new file mode 100644 index 0000000..7b08163 --- /dev/null +++ b/.idea/.idea.CrestronOpenCvSharp/.idea/indexLayout.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/CrestronOpenCvSharp.sln b/CrestronOpenCvSharp.sln new file mode 100644 index 0000000..0efd1a8 --- /dev/null +++ b/CrestronOpenCvSharp.sln @@ -0,0 +1,16 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CrestronOpenCvSharp", "CrestronOpenCvSharp\CrestronOpenCvSharp.csproj", "{82AA53EC-314A-4435-9C78-F90441031C22}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {82AA53EC-314A-4435-9C78-F90441031C22}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {82AA53EC-314A-4435-9C78-F90441031C22}.Debug|Any CPU.Build.0 = Debug|Any CPU + {82AA53EC-314A-4435-9C78-F90441031C22}.Release|Any CPU.ActiveCfg = Release|Any CPU + {82AA53EC-314A-4435-9C78-F90441031C22}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection +EndGlobal diff --git a/CrestronOpenCvSharp/.DS_Store b/CrestronOpenCvSharp/.DS_Store new file mode 100644 index 0000000..a6f738b Binary files /dev/null and b/CrestronOpenCvSharp/.DS_Store differ diff --git a/CrestronOpenCvSharp/Capture/FacialRecognition.cs b/CrestronOpenCvSharp/Capture/FacialRecognition.cs new file mode 100644 index 0000000..30eee1e --- /dev/null +++ b/CrestronOpenCvSharp/Capture/FacialRecognition.cs @@ -0,0 +1,116 @@ +using FaceAiSharp; +using FaceAiSharp.Extensions; +using SixLabors.ImageSharp; +using SixLabors.ImageSharp.PixelFormats; + +namespace CrestronOpenCvSharp.Capture; + +public class FacialRecognition +{ + private readonly IFaceDetectorWithLandmarks _detector; + private readonly IFaceEmbeddingsGenerator _recognizer; + private readonly string? _baseDirectory; + + private Image? _image; + private float[]? _referenceEmbeddings; + + private readonly Dictionary _faceImagesDict; + private string? FaceImagePath { get; set; } + + public FacialRecognition(string? baseDirectory) + { + _baseDirectory = baseDirectory; + _detector = FaceAiSharpBundleFactory.CreateFaceDetectorWithLandmarks(); + _recognizer = FaceAiSharpBundleFactory.CreateFaceEmbeddingsGenerator(); + + if (_baseDirectory != null) + FaceImagePath = Path.Combine(_baseDirectory, "aligned.png"); + + // Let's load the default stuff in this dictionary + _faceImagesDict = new Dictionary + { + { "Yuri Staal", "https://ise2025.local.staal.one/VirtualControl/MA/Rooms/MYFIRSTAI/Html/yuri.jpg" }, + { "Toine C. Leerentveld", "https://ise2025.local.staal.one/VirtualControl/MA/Rooms/MYFIRSTAI/Html/toine.jpg" }, + { "Oliver Hall", "https://ise2025.local.staal.one/VirtualControl/MA/Rooms/MYFIRSTAI/Html/oliver.jpg" } + }; + } + + public bool CheckForFace(string imageFilePath) + { + + try + { + // Load the photo + var photo = File.ReadAllBytes(imageFilePath); + // Convert it + _image = Image.Load(photo); + // Detect faces in this photo + var faces = _detector.DetectFaces(_image); + + if (faces.Count != 0) + { + _recognizer.AlignFaceUsingLandmarks(_image, faces.First().Landmarks!); + _referenceEmbeddings = _recognizer.GenerateEmbedding(_image); + _image.Save(FaceImagePath!); + Console.WriteLine("Aligned faces!"); + } + else + { + Console.WriteLine("No faces were found!"); + } + + // Return true or false + return faces.Any(); + } + catch (Exception e) + { + Console.WriteLine($"Exception detecting faces: {e.Message}"); + throw; + } + } + + public async Task CompareFaces() + { + foreach (var (name, value) in _faceImagesDict) + { + var faceImage = await LoadImageAsync(value); + var detectedFace = _detector.DetectFaces(faceImage).FirstOrDefault(); + + // Generate embedding for the detected face + _recognizer.AlignFaceUsingLandmarks(faceImage, detectedFace.Landmarks!); + var faceEmbedding = _recognizer.GenerateEmbedding(faceImage); + + // Compare embeddings + var similarity = _referenceEmbeddings?.Dot(faceEmbedding); + Console.WriteLine($"Similarity with {name}: {similarity}"); + if (similarity >= 0.42) + { + //Console.WriteLine("Assessment: Both pictures show the same person."); + return name; + } + } + return null; + } + + public void AddPersonToDatabase(string name) + { + var shortName = name.Replace(" ", ""); + // Copy the aligned image to a new image + if (_baseDirectory != null) + { + var newFile = Path.Combine(_baseDirectory, $"{shortName}.jpg"); + Console.WriteLine($"Saved new image to {newFile}"); + File.Copy(FaceImagePath!, newFile, overwrite: true); + } + + _faceImagesDict.Add(name, $"https://ise2025.local.staal.one/VirtualControl/MA/Rooms/MYFIRSTAI/Html/{shortName}.jpg"); + Console.WriteLine($"Added new image to dictionary"); + } + + private async Task> LoadImageAsync(string path) + { + using var hc = new HttpClient(); + var imageBytes = await hc.GetByteArrayAsync(path); + return Image.Load(imageBytes); + } +} \ No newline at end of file diff --git a/CrestronOpenCvSharp/Capture/MjpegCapture.cs b/CrestronOpenCvSharp/Capture/MjpegCapture.cs new file mode 100644 index 0000000..63549cf --- /dev/null +++ b/CrestronOpenCvSharp/Capture/MjpegCapture.cs @@ -0,0 +1,129 @@ +using RandomNameGeneratorLibrary; + +namespace CrestronOpenCvSharp.Capture; + +public class MjpegCapture +{ + private readonly HttpClient _client = new HttpClient(); + private readonly string? _mjpegUrl; + private readonly string? _directory; + private readonly FacialRecognition? _facialRecognition; + private Timer? _timer; + private readonly PersonNameGenerator _personNameGenerator; + + public bool CaptureRunning { get; private set; } + + public MjpegCapture(string? url, string? directory, FacialRecognition? facialRecognition) + { + _mjpegUrl = url; + _directory = Path.Combine(directory!, "captures"); + _facialRecognition = facialRecognition; + + _personNameGenerator = new PersonNameGenerator(); + } + + public void StartCapture(int timeout) + { + if (string.IsNullOrEmpty(_directory)) + { + Console.WriteLine("Directory variable was null or empty."); + CaptureRunning = false; + return; + } + CheckIfDirectoryExists(_directory); + // Only create if it's null + _timer = new Timer(CaptureImage, null, 0, timeout); + CaptureRunning = true; + } + + public void StopCapture() + { + if (_timer is not null) + { + // Stop the timer + _timer.Change(Timeout.Infinite, Timeout.Infinite); + // Dispose it + _timer.Dispose(); + } + CaptureRunning = false; + } + + private void CheckIfDirectoryExists(string directory) + { + Console.WriteLine($"Checking if directory {directory} exists."); + if (Directory.Exists(directory)) + { + // Remove the directory and all its contents + Directory.Delete(directory, true); + Console.WriteLine($"Directory '{directory}' and all its contents have been removed."); + Directory.CreateDirectory(directory); + Console.WriteLine($"Directory '{directory}' has been created."); + } + else + { + // Create the directory + Directory.CreateDirectory(directory); + Console.WriteLine($"Directory '{directory}' has been created."); + } + } + + private async void CaptureImage(object? state) + { + try + { + CaptureRunning = true; + var fileName = Path.Combine(_directory!, $"frame_{DateTime.Now:yyyyMMdd_HHmmss}.jpg"); + try + { + HttpResponseMessage response = await _client.GetAsync(_mjpegUrl, HttpCompletionOption.ResponseHeadersRead); + response.EnsureSuccessStatusCode(); + + await using (var stream = await response.Content.ReadAsStreamAsync()) + await using (var fileStream = new FileStream(fileName, FileMode.Create, FileAccess.Write, FileShare.None)) + { + await stream.CopyToAsync(fileStream); + } + + Console.WriteLine($"Saved frame to {fileName}"); + // TODO Send to image preview window on touchpanel + + if (_facialRecognition!.CheckForFace(fileName)) + { + StopCapture(); + Console.WriteLine("Face detected, stopping capture"); + // TODO Send image to preview window on touchpanel + + //Path.Combine(_baseDirectory, "aligned.png")); + + Console.WriteLine("Comparing captured face against database"); + var result = await _facialRecognition.CompareFaces(); + if (result is not null) + { + Console.WriteLine($"We have found a match! The person in front of the camera is: {result}"); + } + else + { + Console.WriteLine("No match was found in our database, let's add this person!"); + var randomFullName = _personNameGenerator.GenerateRandomFirstAndLastName(); + _facialRecognition.AddPersonToDatabase(randomFullName); + Thread.Sleep(2000); + Console.WriteLine("Person added to database, restarting capture"); + StartCapture(2000); + } + } + else + { + Console.WriteLine("No face detected, carrying on!"); + } + } + catch (Exception ex) + { + Console.WriteLine($"Error capturing image: {ex.Message}"); + } + } + catch (Exception e) + { + Console.WriteLine($"Error capturing image: {e}"); + } + } +} \ No newline at end of file diff --git a/CrestronOpenCvSharp/ControlSystem.cs b/CrestronOpenCvSharp/ControlSystem.cs new file mode 100644 index 0000000..978e030 --- /dev/null +++ b/CrestronOpenCvSharp/ControlSystem.cs @@ -0,0 +1,48 @@ +using Crestron.SimplSharp; +using Crestron.SimplSharpPro; +using CrestronOpenCvSharp.Capture; +using Directory = Crestron.SimplSharp.CrestronIO.Directory; +using Path = Crestron.SimplSharp.CrestronIO.Path; + +namespace CrestronOpenCvSharp; + +public class ControlSystem : CrestronControlSystem +{ + private FacialRecognition? _facialRecognition; + private MjpegCapture? _capture; + private UiHandler? _uiHandler; + + // Default snapshot URL of a Bosch camera + private const string Url = "http://192.168.1.221/snap.jpg"; + private readonly string? _directory; + + public ControlSystem() : base() + { + try + { + Crestron.SimplSharpPro.CrestronThread.Thread.MaxNumberOfUserThreads = 20; + _directory = Path.Combine(Directory.GetApplicationRootDirectory(), $"html"); + Console.WriteLine($"Home directory = {_directory}"); + } + catch (Exception e) + { + ErrorLog.Error("Error in the constructor: {0}", e.Message); + } + } + + public override void InitializeSystem() + { + try + { + _facialRecognition = new FacialRecognition(_directory); + _capture = new MjpegCapture(Url, _directory, _facialRecognition); + _uiHandler = new UiHandler(_capture); + + _capture.StartCapture(5000); + } + catch (Exception e) + { + ErrorLog.Error("Error in InitializeSystem: {0}", e.Message); + } + } +} \ No newline at end of file diff --git a/CrestronOpenCvSharp/CrestronOpenCvSharp.csproj b/CrestronOpenCvSharp/CrestronOpenCvSharp.csproj new file mode 100644 index 0000000..16ea2c5 --- /dev/null +++ b/CrestronOpenCvSharp/CrestronOpenCvSharp.csproj @@ -0,0 +1,37 @@ + + + + net8.0 + enable + enable + + + + + + + + + + + + + + + + + + + + + + + + + + + Always + + + + diff --git a/CrestronOpenCvSharp/UiHandler.cs b/CrestronOpenCvSharp/UiHandler.cs new file mode 100644 index 0000000..46b8d36 --- /dev/null +++ b/CrestronOpenCvSharp/UiHandler.cs @@ -0,0 +1,13 @@ +using CrestronOpenCvSharp.Capture; + +namespace CrestronOpenCvSharp; + +public class UiHandler +{ + private MjpegCapture _capture; + + public UiHandler(MjpegCapture capture) + { + _capture = capture; + } +} \ No newline at end of file diff --git a/CrestronOpenCvSharp/libonnxruntime.so b/CrestronOpenCvSharp/libonnxruntime.so new file mode 100644 index 0000000..0ce9417 Binary files /dev/null and b/CrestronOpenCvSharp/libonnxruntime.so differ diff --git a/global.json b/global.json new file mode 100644 index 0000000..2ddda36 --- /dev/null +++ b/global.json @@ -0,0 +1,7 @@ +{ + "sdk": { + "version": "8.0.0", + "rollForward": "latestMinor", + "allowPrerelease": false + } +} \ No newline at end of file