@multiface.js/context
Version:
Context awareness and memory management for multimodal interactions
717 lines (707 loc) • 30 kB
JavaScript
import { create } from 'zustand';
import AsyncStorage from '@react-native-async-storage/async-storage';
import { useState, useRef, useCallback, useEffect } from 'react';
/******************************************************************************
Copyright (c) Microsoft Corporation.
Permission to use, copy, modify, and/or distribute this software for any
purpose with or without fee is hereby granted.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
PERFORMANCE OF THIS SOFTWARE.
***************************************************************************** */
/* global Reflect, Promise, SuppressedError, Symbol, Iterator */
function __awaiter(thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
}
typeof SuppressedError === "function" ? SuppressedError : function (error, suppressed, message) {
var e = new Error(message);
return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
};
class ContextManager {
constructor(config, callbacks = {}) {
this.contextUpdateTimer = null;
this.storageKey = '@multiface_context';
// Set default values and merge with user config
const defaultConfig = {
enableLearning: true,
maxConversationHistory: 100,
contextUpdateInterval: 30000, // 30 seconds
privacyMode: 'balanced',
biometricAuth: false,
locationTracking: true,
conversationSummary: true
};
this.config = Object.assign({}, defaultConfig, config);
this.callbacks = callbacks;
this.initializeStore();
}
initializeStore() {
this.store = create((set, get) => ({
currentContext: this.getDefaultContext(),
userProfile: this.getDefaultUserProfile(),
conversationHistory: [],
activeMemories: [],
contextualIntents: [],
isLearning: this.config.enableLearning,
privacySettings: {
storeConversations: this.config.privacyMode !== 'strict',
shareBiometrics: this.config.biometricAuth,
shareLocation: this.config.locationTracking,
dataRetentionDays: this.config.privacyMode === 'strict' ? 7 :
this.config.privacyMode === 'balanced' ? 30 : 90
},
updateContext: (contextUpdate) => {
set((state) => {
var _a, _b;
const newContext = Object.assign(Object.assign({}, state.currentContext), contextUpdate);
(_b = (_a = this.callbacks).onContextUpdate) === null || _b === void 0 ? void 0 : _b.call(_a, newContext);
// Emit context update event
this.emitEvent({
type: 'context_update',
data: newContext,
timestamp: Date.now()
});
return { currentContext: newContext };
});
},
addConversationMemory: (memory) => {
set((state) => {
var _a, _b;
const newHistory = [memory, ...state.conversationHistory]
.slice(0, this.config.maxConversationHistory);
// Update active memories (last 5 conversations)
const activeMemories = newHistory.slice(0, 5);
(_b = (_a = this.callbacks).onMemoryCreated) === null || _b === void 0 ? void 0 : _b.call(_a, memory);
// Emit memory created event
this.emitEvent({
type: 'memory_created',
data: memory,
timestamp: Date.now()
});
return {
conversationHistory: newHistory,
activeMemories
};
});
},
updateUserProfile: (profileUpdate) => {
set((state) => {
var _a, _b;
const newProfile = Object.assign(Object.assign({}, state.userProfile), profileUpdate);
(_b = (_a = this.callbacks).onUserProfileUpdated) === null || _b === void 0 ? void 0 : _b.call(_a, newProfile);
// Emit profile update event
this.emitEvent({
type: 'profile_updated',
data: newProfile,
timestamp: Date.now()
});
return { userProfile: newProfile };
});
},
detectIntent: (input, context) => {
var _a, _b;
const intent = this.analyzeIntent(input, context);
set((state) => ({
contextualIntents: [intent, ...state.contextualIntents.slice(0, 9)]
}));
(_b = (_a = this.callbacks).onIntentDetected) === null || _b === void 0 ? void 0 : _b.call(_a, intent);
// Emit intent detected event
this.emitEvent({
type: 'intent_detected',
data: intent,
timestamp: Date.now()
});
return intent;
},
getRelevantMemories: (query, limit = 5) => {
const state = get();
return this.searchMemories(state.conversationHistory, query, limit);
},
clearOldMemories: () => {
set((state) => {
const cutoffDate = Date.now() - (state.privacySettings.dataRetentionDays * 24 * 60 * 60 * 1000);
const filteredHistory = state.conversationHistory.filter(memory => memory.timestamp > cutoffDate);
return {
conversationHistory: filteredHistory,
activeMemories: filteredHistory.slice(0, 5)
};
});
}
}));
}
initialize() {
return __awaiter(this, void 0, void 0, function* () {
try {
yield this.loadPersistedData();
this.startContextUpdates();
}
catch (error) {
console.error('Failed to initialize context manager:', error);
}
});
}
loadPersistedData() {
return __awaiter(this, void 0, void 0, function* () {
try {
const storedData = yield AsyncStorage.getItem(this.storageKey);
if (storedData) {
const parsedData = JSON.parse(storedData);
// Update store with persisted data
this.store.getState().updateUserProfile(parsedData.userProfile || {});
if (parsedData.conversationHistory && this.store.getState().privacySettings.storeConversations) {
parsedData.conversationHistory.forEach((memory) => {
this.store.getState().addConversationMemory(memory);
});
}
}
}
catch (error) {
console.error('Failed to load persisted context data:', error);
}
});
}
persistData() {
return __awaiter(this, void 0, void 0, function* () {
try {
const state = this.store.getState();
const dataToStore = {
userProfile: state.userProfile,
conversationHistory: state.privacySettings.storeConversations ?
state.conversationHistory : [],
timestamp: Date.now()
};
yield AsyncStorage.setItem(this.storageKey, JSON.stringify(dataToStore));
}
catch (error) {
console.error('Failed to persist context data:', error);
}
});
}
startContextUpdates() {
if (this.contextUpdateTimer) {
clearInterval(this.contextUpdateTimer);
}
this.contextUpdateTimer = setInterval(() => {
this.updateTemporalContext();
this.persistData();
}, this.config.contextUpdateInterval);
}
updateTemporalContext() {
const now = new Date();
const hour = now.getHours();
let timeOfDay;
if (hour < 6)
timeOfDay = 'night';
else if (hour < 12)
timeOfDay = 'morning';
else if (hour < 18)
timeOfDay = 'afternoon';
else
timeOfDay = 'evening';
const temporalUpdate = {
temporal: {
timestamp: Date.now(),
timeOfDay,
dayOfWeek: now.toLocaleDateString('en-US', { weekday: 'long' }),
isWeekend: now.getDay() === 0 || now.getDay() === 6,
timezone: Intl.DateTimeFormat().resolvedOptions().timeZone
}
};
this.store.getState().updateContext(temporalUpdate);
}
analyzeIntent(input, context) {
const lowerInput = input.toLowerCase();
// Simple intent detection (in a real implementation, this would use NLP/ML)
const intentPatterns = {
'get_weather': ['weather', 'temperature', 'rain', 'sunny', 'cloudy'],
'set_reminder': ['remind', 'reminder', 'schedule', 'appointment'],
'play_music': ['play', 'music', 'song', 'artist', 'album'],
'navigation': ['navigate', 'directions', 'route', 'go to', 'find'],
'call_contact': ['call', 'phone', 'dial', 'contact'],
'send_message': ['message', 'text', 'send', 'sms'],
'search': ['search', 'find', 'look up', 'google'],
'control_device': ['turn on', 'turn off', 'dim', 'brighten', 'volume']
};
let detectedIntent = 'unknown';
let confidence = 0.3;
for (const [intent, patterns] of Object.entries(intentPatterns)) {
const matches = patterns.filter(pattern => lowerInput.includes(pattern));
if (matches.length > 0) {
detectedIntent = intent;
confidence = Math.min(0.9, 0.5 + (matches.length * 0.2));
break;
}
}
// Extract entities (simplified)
const entities = this.extractEntities(input);
// Generate contextual suggestions
const suggestions = this.generateSuggestions(detectedIntent, context);
return {
intent: detectedIntent,
confidence,
entities,
context: {
temporal: context.temporal.timeOfDay !== undefined,
spatial: context.spatial.location !== undefined,
personal: true,
conversational: this.store.getState().activeMemories.length > 0
},
suggestions
};
}
extractEntities(input) {
const entities = [];
// Time entities
const timePatterns = [
{ pattern: /(\d{1,2}:\d{2})/g, type: 'time' },
{ pattern: /(tomorrow|today|yesterday)/gi, type: 'date' },
{ pattern: /(morning|afternoon|evening|night)/gi, type: 'time_period' }
];
// Location entities
const locationPatterns = [
{ pattern: /(home|work|office|school)/gi, type: 'location' },
{ pattern: /(\d+\s+\w+\s+(street|road|avenue|blvd))/gi, type: 'address' }
];
[...timePatterns, ...locationPatterns].forEach(({ pattern, type }) => {
const matches = input.match(pattern);
if (matches) {
matches.forEach(match => {
entities.push({
type,
value: match.trim(),
confidence: 0.8
});
});
}
});
return entities;
}
generateSuggestions(intent, context) {
const suggestions = [];
switch (intent) {
case 'get_weather':
suggestions.push('What\'s the weather like today?');
if (context.spatial.location) {
suggestions.push('Weather forecast for this week');
}
break;
case 'set_reminder':
suggestions.push('Set reminder for tomorrow');
if (context.temporal.timeOfDay === 'evening') {
suggestions.push('Remind me tomorrow morning');
}
break;
case 'play_music':
suggestions.push('Play my favorite playlist');
if (context.user.activityLevel === 'running') {
suggestions.push('Play workout music');
}
break;
case 'navigation':
suggestions.push('Navigate to home');
suggestions.push('Find nearby restaurants');
break;
default:
suggestions.push('What can I help you with?');
suggestions.push('Try asking about weather, music, or reminders');
}
return suggestions.slice(0, 3);
}
searchMemories(memories, query, limit) {
const lowerQuery = query.toLowerCase();
return memories
.filter(memory => {
var _a;
return ((_a = memory.summary) === null || _a === void 0 ? void 0 : _a.toLowerCase().includes(lowerQuery)) ||
memory.topics.some(topic => topic.toLowerCase().includes(lowerQuery)) ||
memory.messages.some(msg => msg.content.toLowerCase().includes(lowerQuery));
})
.sort((a, b) => b.importance - a.importance)
.slice(0, limit);
}
getDefaultContext() {
const now = new Date();
const hour = now.getHours();
let timeOfDay;
if (hour < 6)
timeOfDay = 'night';
else if (hour < 12)
timeOfDay = 'morning';
else if (hour < 18)
timeOfDay = 'afternoon';
else
timeOfDay = 'evening';
return {
temporal: {
timestamp: Date.now(),
timeOfDay,
dayOfWeek: now.toLocaleDateString('en-US', { weekday: 'long' }),
isWeekend: now.getDay() === 0 || now.getDay() === 6,
timezone: Intl.DateTimeFormat().resolvedOptions().timeZone
},
spatial: {
environment: 'unknown',
ambientLight: 'unknown',
noiseLevel: 'unknown'
},
device: {
networkType: 'wifi',
orientation: 'portrait'
},
app: {
currentScreen: 'main',
sessionDuration: 0
},
user: {
activityLevel: 'stationary',
attentionLevel: 'focused'
}
};
}
getDefaultUserProfile() {
return {
id: `user_${Date.now()}`,
preferences: {
language: 'en-US',
voiceSettings: {
speechRate: 1.0,
volume: 0.8
},
interactionModes: ['voice', 'text', 'gesture'],
personalizations: {}
},
usage: {
totalInteractions: 0,
preferredInputTypes: {},
commonCommands: [],
lastActiveTime: Date.now()
}
};
}
emitEvent(event) {
// In a real implementation, this could emit to an event bus
console.log('Context Event:', event);
}
// Public API methods
getContext() {
return this.store.getState().currentContext;
}
getUserProfile() {
return this.store.getState().userProfile;
}
addMessage(content, type = 'text') {
const conversationId = `conv_${Date.now()}`;
const message = {
id: `msg_${Date.now()}`,
sender: 'user',
content,
type,
timestamp: Date.now()
};
const memory = {
id: conversationId,
timestamp: Date.now(),
participants: ['user', 'assistant'],
messages: [message],
context: this.getContext(),
topics: this.extractTopics(content),
sentiment: this.analyzeSentiment(content),
importance: this.calculateImportance(content, this.getContext())
};
this.store.getState().addConversationMemory(memory);
}
extractTopics(content) {
// Simple topic extraction (in production, use NLP)
const words = content.toLowerCase().split(/\s+/);
const stopWords = ['the', 'a', 'an', 'and', 'or', 'but', 'in', 'on', 'at', 'to', 'for', 'of', 'with', 'by'];
return words.filter(word => word.length > 3 && !stopWords.includes(word)).slice(0, 5);
}
analyzeSentiment(content) {
// Simple sentiment analysis
const positiveWords = ['good', 'great', 'excellent', 'happy', 'love', 'like', 'amazing'];
const negativeWords = ['bad', 'terrible', 'hate', 'dislike', 'awful', 'horrible', 'sad'];
const lowerContent = content.toLowerCase();
const positiveCount = positiveWords.filter(word => lowerContent.includes(word)).length;
const negativeCount = negativeWords.filter(word => lowerContent.includes(word)).length;
if (positiveCount > negativeCount)
return 'positive';
if (negativeCount > positiveCount)
return 'negative';
return 'neutral';
}
calculateImportance(content, context) {
let importance = 5; // Base importance
// Increase importance for certain keywords
const importantKeywords = ['urgent', 'important', 'emergency', 'reminder', 'appointment'];
const hasImportantKeywords = importantKeywords.some(keyword => content.toLowerCase().includes(keyword));
if (hasImportantKeywords)
importance += 3;
// Increase importance during work hours
if (context.temporal.timeOfDay === 'morning' || context.temporal.timeOfDay === 'afternoon') {
importance += 1;
}
return Math.min(10, importance);
}
updateSpatialContext(location, environment) {
const spatialUpdate = {
spatial: Object.assign(Object.assign(Object.assign({}, this.getContext().spatial), (location && { location })), (environment && { environment: environment }))
};
this.store.getState().updateContext(spatialUpdate);
}
updateDeviceContext(deviceInfo) {
const deviceUpdate = {
device: Object.assign(Object.assign({}, this.getContext().device), deviceInfo)
};
this.store.getState().updateContext(deviceUpdate);
}
updateUserActivity(activityLevel) {
const userUpdate = {
user: Object.assign(Object.assign({}, this.getContext().user), { activityLevel })
};
this.store.getState().updateContext(userUpdate);
}
stop() {
if (this.contextUpdateTimer) {
clearInterval(this.contextUpdateTimer);
this.contextUpdateTimer = null;
}
this.persistData();
}
}
const useContext = (options = {}) => {
const defaultConfig = {
enableLearning: true,
maxConversationHistory: 50,
contextUpdateInterval: 30000,
privacyMode: 'balanced',
biometricAuth: false,
locationTracking: true,
conversationSummary: true
};
const { config = defaultConfig, autoStart = true, enableFusion = false, enableSensorIntegration = false, onFusedOutput } = options;
const [contextState, setContextState] = useState({
currentContext: {},
userProfile: {},
conversationHistory: [],
activeMemories: [],
contextualIntents: [],
isLearning: true,
privacySettings: {
storeConversations: true,
shareBiometrics: false,
shareLocation: true,
dataRetentionDays: 30
}
});
const [lastIntent, setLastIntent] = useState(null);
const contextManagerRef = useRef(null);
// Integration placeholders for fusion and sensor systems
const processInput = enableFusion ? console.log : undefined;
const sensorState = { isActive: false };
const callbacks = {
onContextUpdate: useCallback((context) => {
setContextState(prev => (Object.assign(Object.assign({}, prev), { currentContext: context })));
// Send context updates to fusion system if enabled
if (enableFusion && processInput) {
processInput('Context update:', context);
}
}, [enableFusion, processInput]),
onIntentDetected: useCallback((intent) => {
setLastIntent(intent);
setContextState(prev => (Object.assign(Object.assign({}, prev), { contextualIntents: [intent, ...prev.contextualIntents.slice(0, 9)] })));
}, []),
onMemoryCreated: useCallback((memory) => {
setContextState(prev => (Object.assign(Object.assign({}, prev), { conversationHistory: [memory, ...prev.conversationHistory], activeMemories: [memory, ...prev.activeMemories.slice(0, 4)] })));
}, []),
onUserProfileUpdated: useCallback((profile) => {
setContextState(prev => (Object.assign(Object.assign({}, prev), { userProfile: profile })));
}, []),
onPrivacyAlert: useCallback((alert, data) => {
console.warn('Privacy Alert:', alert, data);
}, [])
};
const initializeContext = useCallback(() => __awaiter(void 0, void 0, void 0, function* () {
if (contextManagerRef.current) {
contextManagerRef.current.stop();
}
contextManagerRef.current = new ContextManager(config, callbacks);
try {
yield contextManagerRef.current.initialize();
// Update state with initial context
setContextState(prev => (Object.assign(Object.assign({}, prev), { currentContext: contextManagerRef.current.getContext(), userProfile: contextManagerRef.current.getUserProfile() })));
}
catch (error) {
console.error('Failed to initialize context manager:', error);
}
}), [config, callbacks]);
const addMessage = useCallback((content, type = 'text') => {
var _a;
(_a = contextManagerRef.current) === null || _a === void 0 ? void 0 : _a.addMessage(content, type);
}, []);
const detectIntent = useCallback((input) => {
if (!contextManagerRef.current)
return null;
contextManagerRef.current.getContext();
// detectIntent method will be available when ContextManager is fully implemented
return null;
}, []);
const updateSpatialContext = useCallback((location, environment) => {
var _a;
(_a = contextManagerRef.current) === null || _a === void 0 ? void 0 : _a.updateSpatialContext(location, environment);
}, []);
const updateDeviceContext = useCallback((deviceInfo) => {
var _a;
(_a = contextManagerRef.current) === null || _a === void 0 ? void 0 : _a.updateDeviceContext(deviceInfo);
}, []);
const updateUserActivity = useCallback((activityLevel) => {
var _a;
(_a = contextManagerRef.current) === null || _a === void 0 ? void 0 : _a.updateUserActivity(activityLevel);
}, []);
const getRelevantMemories = useCallback((query, limit = 5) => {
if (!contextManagerRef.current)
return [];
// getRelevantMemories method will be available when ContextManager is fully implemented
return [];
}, []);
const getCurrentContext = useCallback(() => {
var _a;
return ((_a = contextManagerRef.current) === null || _a === void 0 ? void 0 : _a.getContext()) || contextState.currentContext;
}, [contextState.currentContext]);
const getUserProfile = useCallback(() => {
var _a;
return ((_a = contextManagerRef.current) === null || _a === void 0 ? void 0 : _a.getUserProfile()) || contextState.userProfile;
}, [contextState.userProfile]);
// Integrate sensor data with context (simplified for build)
useEffect(() => {
}, [enableSensorIntegration, sensorState.isActive]);
useEffect(() => {
if (autoStart) {
initializeContext();
}
return () => {
var _a;
(_a = contextManagerRef.current) === null || _a === void 0 ? void 0 : _a.stop();
};
}, [autoStart, initializeContext]);
return {
contextState,
lastIntent,
addMessage,
detectIntent,
updateSpatialContext,
updateDeviceContext,
updateUserActivity,
getRelevantMemories,
getCurrentContext,
getUserProfile,
isInitialized: !!contextManagerRef.current
};
};
// Specialized hook for conversation memory
const useConversationMemory = (maxMemories = 10) => {
const defaultConfig = {
maxConversationHistory: maxMemories,
enableLearning: true,
contextUpdateInterval: 30000,
privacyMode: 'balanced',
biometricAuth: false,
locationTracking: true,
conversationSummary: true
};
const { contextState, addMessage, getRelevantMemories } = useContext({
config: defaultConfig
});
const addConversation = useCallback((userMessage, assistantResponse) => {
addMessage(userMessage, 'text');
// In a real implementation, you'd also add the assistant response
}, [addMessage]);
const searchMemories = useCallback((query) => {
return getRelevantMemories(query, 5);
}, [getRelevantMemories]);
return {
conversationHistory: contextState.conversationHistory,
activeMemories: contextState.activeMemories,
addConversation,
searchMemories
};
};
// Specialized hook for intent detection
const useIntentDetection = (onIntentDetected) => {
const { lastIntent, detectIntent, getCurrentContext } = useContext();
const analyzeInput = useCallback((input) => {
const intent = detectIntent(input);
if (intent && onIntentDetected) {
onIntentDetected(intent);
}
return intent;
}, [detectIntent, onIntentDetected]);
const getContextualSuggestions = useCallback(() => {
var _a;
const context = getCurrentContext();
const timeOfDay = (_a = context.temporal) === null || _a === void 0 ? void 0 : _a.timeOfDay;
const suggestions = ['What can I help you with?'];
if (timeOfDay === 'morning') {
suggestions.push('Check today\'s weather', 'Review my schedule');
}
else if (timeOfDay === 'evening') {
suggestions.push('Set a reminder', 'Play some music');
}
return suggestions;
}, [getCurrentContext]);
return {
lastIntent,
analyzeInput,
getContextualSuggestions
};
};
// Specialized hook for user personalization
const usePersonalization = () => {
const defaultConfig = {
enableLearning: true,
maxConversationHistory: 50,
contextUpdateInterval: 30000,
privacyMode: 'balanced',
biometricAuth: false,
locationTracking: true,
conversationSummary: true
};
const { contextState, getCurrentContext, getUserProfile } = useContext({
config: defaultConfig
});
const getPersonalizedResponse = useCallback((baseResponse) => {
var _a, _b;
getUserProfile();
const context = getCurrentContext();
// Simple personalization based on time and preferences
let personalizedResponse = baseResponse;
if (((_a = context.temporal) === null || _a === void 0 ? void 0 : _a.timeOfDay) === 'morning') {
personalizedResponse = `Good morning! ${baseResponse}`;
}
else if (((_b = context.temporal) === null || _b === void 0 ? void 0 : _b.timeOfDay) === 'evening') {
personalizedResponse = `Good evening! ${baseResponse}`;
}
return personalizedResponse;
}, [getUserProfile, getCurrentContext]);
const updatePreferences = useCallback((preferences) => {
// This would update user preferences in the context manager
console.log('Updating preferences:', preferences);
}, []);
return {
userProfile: contextState.userProfile,
getPersonalizedResponse,
updatePreferences,
usageStats: contextState.userProfile.usage
};
};
export { ContextManager, useContext, useConversationMemory, useIntentDetection, usePersonalization };
//# sourceMappingURL=index.esm.js.map