From a8751ca728748c39739df994bb002e6501ef04e9 Mon Sep 17 00:00:00 2001 From: Marc Rejohn Castillano Date: Sun, 8 Mar 2026 10:19:03 +0800 Subject: [PATCH] Face Recognition with Liveness Detection for Web Support --- lib/screens/attendance/attendance_screen.dart | 3 +- lib/screens/profile/profile_screen.dart | 29 +- lib/services/face_verification_mobile.dart | 26 ++ lib/services/face_verification_web.dart | 220 +++------- lib/widgets/face_verification_overlay.dart | 31 +- web/face_interop.js | 399 ++++++++++++++++++ web/index.html | 1 + 7 files changed, 513 insertions(+), 196 deletions(-) create mode 100644 web/face_interop.js diff --git a/lib/screens/attendance/attendance_screen.dart b/lib/screens/attendance/attendance_screen.dart index 0db51f5b..8ea0aca6 100644 --- a/lib/screens/attendance/attendance_screen.dart +++ b/lib/screens/attendance/attendance_screen.dart @@ -1511,8 +1511,9 @@ class _LogbookTab extends ConsumerWidget { if (logScheduleIds.contains(s.id)) return false; if (!s.endTime.isBefore(now)) return false; if (s.startTime.isBefore(range.start) || - !s.startTime.isBefore(range.end)) + !s.startTime.isBefore(range.end)) { return false; + } // If the user rendered overtime on this calendar day, don't // mark the normal/night shift schedule as absent. final d = s.startTime; diff --git a/lib/screens/profile/profile_screen.dart b/lib/screens/profile/profile_screen.dart index fc07a3ad..b8e50747 100644 --- a/lib/screens/profile/profile_screen.dart +++ b/lib/screens/profile/profile_screen.dart @@ -9,6 +9,7 @@ import '../../providers/profile_provider.dart'; import '../../providers/tickets_provider.dart'; import '../../providers/user_offices_provider.dart'; import '../../services/face_verification.dart' as face; +import '../../widgets/face_verification_overlay.dart'; import '../../widgets/multi_select_picker.dart'; import '../../widgets/qr_verification_dialog.dart'; import '../../widgets/responsive_body.dart'; @@ -525,25 +526,19 @@ class _ProfileScreenState extends ConsumerState { } try { - final result = await face.runFaceLiveness(context); - if (result == null || !mounted) return; + final result = await showFaceVerificationOverlay( + context: context, + ref: ref, + // Profile test mode: no attendance record should be uploaded/skipped. + attendanceLogId: null, + uploadAttendanceResult: false, + maxAttempts: 3, + ); - // Download enrolled photo via Supabase (authenticated, private bucket) - final enrolledBytes = await ref - .read(profileControllerProvider) - .downloadFacePhoto(profile.id); - if (enrolledBytes == null || !mounted) { - if (mounted) { - showErrorSnackBar(context, 'Could not load enrolled face photo.'); - } - return; - } + if (!mounted || result == null) return; - // Compare captured vs enrolled - final score = await face.compareFaces(result.imageBytes, enrolledBytes); - - if (!mounted) return; - if (score >= 0.60) { + final score = result.matchScore ?? 0.0; + if (result.verified) { showSuccessSnackBar( context, 'Face matched! Similarity: ${(score * 100).toStringAsFixed(1)}%', diff --git a/lib/services/face_verification_mobile.dart b/lib/services/face_verification_mobile.dart index b8b80ac9..2300fcc1 100644 --- a/lib/services/face_verification_mobile.dart +++ b/lib/services/face_verification_mobile.dart @@ -21,6 +21,9 @@ Future runFaceLiveness( }) async { String? capturedPath; + final colors = Theme.of(context).colorScheme; + final textTheme = Theme.of(context).textTheme; + await Navigator.of(context).push( MaterialPageRoute( builder: (ctx) => LivenessCheckScreen( @@ -35,10 +38,33 @@ Future runFaceLiveness( // Don't pop in onCancel/onError — the package's AppBar // already calls Navigator.pop() after invoking these. ), + // Remove the default placeholder from inside the camera circle; + // it is shown below via customBottomWidget instead. + placeholder: null, + theme: LivenessCheckTheme( + backgroundColor: colors.surface, + overlayColor: colors.surface.withAlpha(230), + primaryColor: colors.primary, + borderColor: colors.primary, + textColor: colors.onSurface, + errorColor: colors.error, + successColor: colors.tertiary, + ), settings: LivenessCheckSettings( requiredBlinkCount: requiredBlinks, requireSmile: false, autoNavigateOnSuccess: false, + // Must be false so that customBottomWidget is shown. + showTryAgainButton: false, + ), + // Challenge instruction rendered below the camera circle. + customBottomWidget: Padding( + padding: const EdgeInsets.fromLTRB(24, 4, 24, 24), + child: Text( + 'Blink $requiredBlinks times or smile naturally to continue', + textAlign: TextAlign.center, + style: textTheme.bodyLarge?.copyWith(color: colors.onSurface), + ), ), ), ), diff --git a/lib/services/face_verification_web.dart b/lib/services/face_verification_web.dart index d1878a8e..beebc96e 100644 --- a/lib/services/face_verification_web.dart +++ b/lib/services/face_verification_web.dart @@ -1,7 +1,6 @@ import 'dart:convert'; import 'dart:js_interop'; import 'dart:typed_data'; -import 'dart:ui_web' as ui_web; import 'package:flutter/material.dart'; @@ -10,20 +9,11 @@ import 'package:flutter/material.dart'; @JS() external JSPromise initFaceApi(); +/// Runs liveness detection via a JS-managed fullscreen overlay. +/// No containerId needed — the JS code appends the overlay to document.body +/// directly, which avoids CanvasKit iframe cross-origin restrictions. @JS() -external JSObject createFaceContainer(JSString id); - -@JS() -external JSPromise startWebCamera(JSString containerId); - -@JS() -external void stopWebCamera(JSString containerId); - -@JS() -external JSPromise runWebLiveness( - JSString containerId, - JSNumber requiredBlinks, -); +external JSPromise runWebLiveness(JSNumber requiredBlinks); @JS() external void cancelWebLiveness(); @@ -103,8 +93,6 @@ Future compareFaces( // ─── Web Liveness Dialog ──────────────────────────────────────────────────── -bool _viewFactoryRegistered = false; - class _WebLivenessDialog extends StatefulWidget { final int requiredBlinks; const _WebLivenessDialog({required this.requiredBlinks}); @@ -113,88 +101,36 @@ class _WebLivenessDialog extends StatefulWidget { State<_WebLivenessDialog> createState() => _WebLivenessDialogState(); } -enum _WebLivenessState { loading, cameraStarting, detecting, error } +enum _WebLivenessState { loading, error } class _WebLivenessDialogState extends State<_WebLivenessDialog> { - late final String _containerId; - late final String _viewType; _WebLivenessState _state = _WebLivenessState.loading; - String _statusText = 'Loading face detection models...'; - int _blinkCount = 0; + String _statusText = 'Loading face detection models…'; String? _errorText; bool _popped = false; @override void initState() { super.initState(); - _containerId = 'face-cam-${DateTime.now().millisecondsSinceEpoch}'; - _viewType = 'web-face-cam-$_containerId'; - - // Register a unique platform view factory for this dialog instance - if (!_viewFactoryRegistered) { - _viewFactoryRegistered = true; - } - ui_web.platformViewRegistry.registerViewFactory(_viewType, ( - int viewId, { - Object? params, - }) { - return createFaceContainer(_containerId.toJS); - }); - WidgetsBinding.instance.addPostFrameCallback((_) => _initialize()); } Future _initialize() async { - try { - // Load face-api.js models - final loaded = await initFaceApi().toDart; - if (!loaded.toDart) { - _setError('Failed to load face detection models.'); - return; - } - - if (!mounted) return; - setState(() { - _state = _WebLivenessState.cameraStarting; - _statusText = 'Starting camera...'; - }); - - // Give the platform view a moment to render - await Future.delayed(const Duration(milliseconds: 500)); - - // Start camera - final cameraStarted = await startWebCamera(_containerId.toJS).toDart; - if (!cameraStarted.toDart) { - _setError( - 'Camera access denied or unavailable.\n' - 'Please allow camera access and try again.', - ); - return; - } - - if (!mounted) return; - setState(() { - _state = _WebLivenessState.detecting; - _statusText = 'Look at the camera and blink naturally'; - _blinkCount = 0; - }); - - // Start liveness detection - _runLiveness(); - } catch (e) { - _setError('Initialization failed: $e'); - } + // initFaceApi() immediately returns true and starts background loading of + // face-api.js (needed for compareFaces later). MediaPipe is initialized + // inside runWebLiveness() itself, with progress shown in the JS overlay. + await initFaceApi().toDart; + if (!mounted) return; + _runLiveness(); } Future _runLiveness() async { try { - final result = await runWebLiveness( - _containerId.toJS, - widget.requiredBlinks.toJS, - ).toDart; + // runWebLiveness opens its own fullscreen JS overlay so the camera video + // element lives in the top-level document — not inside a CanvasKit iframe. + final result = await runWebLiveness(widget.requiredBlinks.toJS).toDart; if (result == null) { - // Cancelled — _cancel() may have already popped if (mounted && !_popped) { _popped = true; Navigator.of(context).pop(null); @@ -204,8 +140,6 @@ class _WebLivenessDialogState extends State<_WebLivenessDialog> { final jsResult = result as _LivenessJSResult; final dataUrl = jsResult.dataUrl.toDart; - - // Convert data URL to bytes final base64Data = dataUrl.split(',')[1]; final bytes = base64Decode(base64Data); @@ -232,16 +166,14 @@ class _WebLivenessDialogState extends State<_WebLivenessDialog> { if (_popped) return; _popped = true; cancelWebLiveness(); - stopWebCamera(_containerId.toJS); Navigator.of(context).pop(null); } void _retry() { setState(() { _state = _WebLivenessState.loading; - _statusText = 'Loading face detection models...'; + _statusText = 'Loading face detection models…'; _errorText = null; - _blinkCount = 0; }); _initialize(); } @@ -249,7 +181,6 @@ class _WebLivenessDialogState extends State<_WebLivenessDialog> { @override void dispose() { cancelWebLiveness(); - stopWebCamera(_containerId.toJS); super.dispose(); } @@ -258,6 +189,39 @@ class _WebLivenessDialogState extends State<_WebLivenessDialog> { final theme = Theme.of(context); final colors = theme.colorScheme; + Widget content; + if (_state == _WebLivenessState.error) { + content = Column( + mainAxisSize: MainAxisSize.min, + children: [ + Icon(Icons.error_outline, color: colors.error, size: 40), + const SizedBox(height: 12), + Text( + _errorText ?? 'An error occurred.', + style: theme.textTheme.bodyMedium?.copyWith(color: colors.error), + textAlign: TextAlign.center, + ), + ], + ); + } else { + content = Column( + mainAxisSize: MainAxisSize.min, + children: [ + const SizedBox( + width: 28, + height: 28, + child: CircularProgressIndicator(strokeWidth: 2.5), + ), + const SizedBox(height: 14), + Text( + _statusText, + style: theme.textTheme.bodyMedium, + textAlign: TextAlign.center, + ), + ], + ); + } + return AlertDialog( shape: RoundedRectangleBorder(borderRadius: BorderRadius.circular(28)), title: Row( @@ -267,89 +231,9 @@ class _WebLivenessDialogState extends State<_WebLivenessDialog> { const Expanded(child: Text('Face Verification')), ], ), - content: ConstrainedBox( - constraints: const BoxConstraints(maxWidth: 400, maxHeight: 480), - child: Column( - mainAxisSize: MainAxisSize.min, - children: [ - // Camera preview - ClipRRect( - borderRadius: BorderRadius.circular(12), - child: SizedBox( - width: 320, - height: 240, - child: _state == _WebLivenessState.error - ? Container( - color: colors.errorContainer, - child: Center( - child: Icon( - Icons.videocam_off, - size: 48, - color: colors.onErrorContainer, - ), - ), - ) - : HtmlElementView(viewType: _viewType), - ), - ), - const SizedBox(height: 16), - - // Status - if (_state == _WebLivenessState.loading || - _state == _WebLivenessState.cameraStarting) - Column( - children: [ - const SizedBox( - width: 24, - height: 24, - child: CircularProgressIndicator(strokeWidth: 2), - ), - const SizedBox(height: 8), - Text( - _statusText, - style: theme.textTheme.bodyMedium, - textAlign: TextAlign.center, - ), - ], - ) - else if (_state == _WebLivenessState.detecting) - Column( - children: [ - Text( - _statusText, - style: theme.textTheme.bodyMedium, - textAlign: TextAlign.center, - ), - const SizedBox(height: 8), - LinearProgressIndicator( - value: _blinkCount / widget.requiredBlinks, - borderRadius: BorderRadius.circular(4), - ), - const SizedBox(height: 4), - Text( - 'Blinks: $_blinkCount / ${widget.requiredBlinks}', - style: theme.textTheme.bodySmall?.copyWith( - color: colors.onSurfaceVariant, - ), - ), - ], - ) - else if (_state == _WebLivenessState.error) - Column( - children: [ - Icon(Icons.error_outline, color: colors.error, size: 32), - const SizedBox(height: 8), - Text( - _errorText ?? 'An error occurred.', - style: theme.textTheme.bodyMedium?.copyWith( - color: colors.error, - ), - textAlign: TextAlign.center, - ), - ], - ), - ], - ), + content: Padding( + padding: const EdgeInsets.symmetric(vertical: 8), + child: content, ), actions: [ if (_state == _WebLivenessState.error) ...[ diff --git a/lib/widgets/face_verification_overlay.dart b/lib/widgets/face_verification_overlay.dart index 78a847a1..de25f13e 100644 --- a/lib/widgets/face_verification_overlay.dart +++ b/lib/widgets/face_verification_overlay.dart @@ -26,8 +26,9 @@ class FaceVerificationResult { Future showFaceVerificationOverlay({ required BuildContext context, required WidgetRef ref, - required String attendanceLogId, + String? attendanceLogId, int maxAttempts = 3, + bool uploadAttendanceResult = true, }) { return Navigator.of(context).push( PageRouteBuilder( @@ -35,6 +36,7 @@ Future showFaceVerificationOverlay({ pageBuilder: (ctx, anim, secAnim) => _FaceVerificationOverlay( attendanceLogId: attendanceLogId, maxAttempts: maxAttempts, + uploadAttendanceResult: uploadAttendanceResult, ), transitionsBuilder: (ctx, anim, secAnim, child) { return FadeTransition(opacity: anim, child: child); @@ -49,10 +51,12 @@ class _FaceVerificationOverlay extends ConsumerStatefulWidget { const _FaceVerificationOverlay({ required this.attendanceLogId, required this.maxAttempts, + required this.uploadAttendanceResult, }); - final String attendanceLogId; + final String? attendanceLogId; final int maxAttempts; + final bool uploadAttendanceResult; @override ConsumerState<_FaceVerificationOverlay> createState() => @@ -108,12 +112,14 @@ class _FaceVerificationOverlayState if (result == null) { if (!mounted) return; // Cancelled: on web offer QR, otherwise mark cancelled - if (kIsWeb) { + if (kIsWeb && + widget.uploadAttendanceResult && + widget.attendanceLogId != null) { final completed = await showQrVerificationDialog( context: context, ref: ref, type: 'verification', - contextId: widget.attendanceLogId, + contextId: widget.attendanceLogId!, ); if (mounted) { Navigator.of( @@ -213,11 +219,14 @@ class _FaceVerificationOverlayState } Future _uploadResult(Uint8List bytes, String status) async { + if (!widget.uploadAttendanceResult || widget.attendanceLogId == null) { + return; + } try { await ref .read(attendanceControllerProvider) .uploadVerification( - attendanceId: widget.attendanceLogId, + attendanceId: widget.attendanceLogId!, bytes: bytes, fileName: 'verification.jpg', status: status, @@ -226,11 +235,13 @@ class _FaceVerificationOverlayState } Future _skipVerification(String reason) async { - try { - await ref - .read(attendanceControllerProvider) - .skipVerification(widget.attendanceLogId); - } catch (_) {} + if (widget.uploadAttendanceResult && widget.attendanceLogId != null) { + try { + await ref + .read(attendanceControllerProvider) + .skipVerification(widget.attendanceLogId!); + } catch (_) {} + } if (!mounted) return; setState(() { _phase = _Phase.cancelled; diff --git a/web/face_interop.js b/web/face_interop.js new file mode 100644 index 00000000..ab156811 --- /dev/null +++ b/web/face_interop.js @@ -0,0 +1,399 @@ +// face_interop.js — TasQ web face verification bridge. +// +// Liveness : MediaPipe FaceLandmarker (blend shapes) — blink OR smile. +// Comparison: face-api.js (128-D face descriptors via faceRecognitionNet). +// +// Pinned CDN versions — update these constants when upgrading: +// @mediapipe/tasks-vision 0.10.21 +// @vladmandic/face-api 1 + +'use strict'; + +// ── Shared state ───────────────────────────────────────────────────────────── +let _livenessRunning = false; +let _activeOverlay = null; // for cancelWebLiveness() to clean up eagerly + +// ── face-api.js — lazy-loaded for face descriptor comparison ───────────────── +let _faceApiLoaded = false; +let _faceApiPromise = null; + +async function _ensureFaceApi() { + if (_faceApiLoaded) return; + if (!_faceApiPromise) { + _faceApiPromise = (async () => { + if (!window.faceapi) { + await new Promise((res, rej) => { + const s = document.createElement('script'); + s.src = 'https://cdn.jsdelivr.net/npm/@vladmandic/face-api@1/dist/face-api.js'; + s.onload = res; + s.onerror = () => rej(new Error('Failed to load face-api.js')); + document.head.appendChild(s); + }); + } + const MODEL_URL = 'https://cdn.jsdelivr.net/npm/@vladmandic/face-api@1/model/'; + await faceapi.nets.tinyFaceDetector.loadFromUri(MODEL_URL); + await faceapi.nets.faceLandmark68Net.loadFromUri(MODEL_URL); + await faceapi.nets.faceRecognitionNet.loadFromUri(MODEL_URL); + _faceApiLoaded = true; + })().catch(e => { _faceApiPromise = null; throw e; }); + } + await _faceApiPromise; +} + +/// Called by Dart on dialog open. Begins background face-api load so +/// that descriptors are ready for compareFaces() after liveness succeeds. +async function initFaceApi() { + _ensureFaceApi().catch(e => console.warn('[face-api bg]', e)); + return true; // always succeeds; real errors surface in getFaceDescriptor* +} + +// ── MediaPipe FaceLandmarker — lazy-loaded for liveness ────────────────────── +// Update _MP_VER when a newer release is available on jsDelivr. +const _MP_VER = '0.10.21'; +const _MP_CDN = `https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@${_MP_VER}`; +const _MP_MODEL = 'https://storage.googleapis.com/mediapipe-models/' + + 'face_landmarker/face_landmarker/float16/1/face_landmarker.task'; + +let _faceLandmarker = null; +let _mpPromise = null; + +async function _ensureMediaPipe(onStatus) { + if (_faceLandmarker) return; + if (!_mpPromise) { + _mpPromise = (async () => { + onStatus?.('Loading face detection…'); + const { FaceLandmarker, FilesetResolver } = + await import(`${_MP_CDN}/vision_bundle.mjs`); + + onStatus?.('Initializing model…'); + const fileset = await FilesetResolver.forVisionTasks(`${_MP_CDN}/wasm`); + + const opts = { + outputFaceBlendshapes: true, + runningMode: 'VIDEO', + numFaces: 1, + }; + const mkLandmarker = (delegate) => + FaceLandmarker.createFromOptions(fileset, { + baseOptions: { modelAssetPath: _MP_MODEL, delegate }, + ...opts, + }); + + // Prefer GPU for throughput; fall back to CPU if unavailable. + try { + _faceLandmarker = await mkLandmarker('GPU'); + } catch { + _faceLandmarker = await mkLandmarker('CPU'); + } + })().catch(e => { _mpPromise = null; throw e; }); + } + await _mpPromise; +} + +// ── Liveness overlay — MediaPipe, blink OR smile ───────────────────────────── +/// Creates a fullscreen overlay appended to document.body so that both the +///