TypeScript Types

1// Root metadata structure
2interface MP4EMetadata {
3  schemaVersion: string;            // "2.0"
4  metadataType: string;             // "mp4e"
5
6  videoDetails: VideoDetails;
7  processingStats?: ProcessingStats;
8
9  objects: {
10    registry: ObjectRegistry;       // Keyed by objectId
11    timeline: FrameData[];          // Frame-by-frame positions
12  };
13
14  overlays: Overlay[];
15  rules: UnifiedRule[];
16  layers: Layer[];
17  variables: Variable[];
18
19  plugins: {
20    enabled: string[];              // Plugin types to enable
21    config: Record<string, any>;
22  };
23
24  objectGroups?: ObjectGroup[];
25  scenes?: Scene[];
26
27  videoId?: string;
28  videoMetadata?: VideoMetadata;
29  playerConfig?: PlayerConfig;
30  trackingEvents?: TrackingEventsConfig;
31  seo?: SEOMetadata;
32}
33
34interface VideoDetails {
35  totalFrames?: number | null;
36  totalFramesAnalyzed?: number | null;
37  fps?: number;
38  width?: number;
39  height?: number;
40  duration?: number;
41}
42
43interface SEOMetadata {
44  title: string;
45  description: string;
46  tags: string[];
47  chapters: Chapter[];
48  thumbnail?: {
49    type: 'frame' | 'custom';
50    frame?: number;
51    data?: string;              // Base64 JPEG for custom
52  };
53}
54
55interface Chapter {
56  id: string;
57  title: string;
58  description?: string;
59  startTime: number;
60  endTime: number;
61}

1// Object registry (keyed by objectId)
2type ObjectRegistry = Record<string, ObjectData>;
3
4interface ObjectData {
5  label: string;                    // AI-detected label
6  userLabel?: string;               // User-defined display name
7  confidence: number;               // Detection confidence (0-1)
8
9  // Tracking type
10  trackingType?: 'bbox' | 'polygon' | 'surface';
11  referencePolygon?: number[];      // Flattened coords for polygon tracking
12  surfaceData?: SurfaceData;        // For surface tracking
13
14  // Frame range
15  firstSeenFrame?: number;
16  lastSeenFrame?: number;
17
18  // Quality metrics
19  trackQuality?: number;
20  totalAppearances?: number;
21  occlusionRate?: number;
22
23  // E-commerce
24  buyLink?: string;
25  productId?: string;
26  price?: number;
27  currency?: string;
28  description?: string;
29
30  // AI analysis
31  llmAnalysis?: {
32    style?: string;
33    color?: string;
34    material?: string;
35    setting?: string;
36  };
37
38  // Custom data
39  customData?: Record<string, any>;
40}
41
42interface SurfaceData {
43  referenceFrame: number;
44  referenceCorners: SurfaceCorners;
45  trackingQuality?: 'draft' | 'standard' | 'high';
46}
47
48interface SurfaceCorners {
49  topLeft: { x: number; y: number };
50  topRight: { x: number; y: number };
51  bottomRight: { x: number; y: number };
52  bottomLeft: { x: number; y: number };
53}
54
55// Timeline frame data
56interface FrameData {
57  frame: number;
58  time: number;                     // frame / fps
59  objects: FrameObject[];
60}
61
62interface FrameObject {
63  id: string;                       // Reference to ObjectData
64  bboxXyxy: [number, number, number, number];  // [x1, y1, x2, y2] normalized
65  visibility?: number;              // 0-1
66  trackingConfidence?: number;      // Per-frame confidence
67
68  // For polygon tracking
69  polygon?: number[];               // Flattened coordinates
70
71  // For surface tracking
72  corners?: SurfaceCorners;
73  cornerConfidence?: number;
74}
75
76// Object groups
77interface ObjectGroup {
78  id: string;
79  name: string;
80  objectIds: string[];
81  displaySettings?: DisplaySettings;
82  rules?: UnifiedRule[];
83}
84
85interface DisplaySettings {
86  type: string;               // e.g., "core:product-card"
87  enabled: boolean;
88  hoverEnabled?: boolean;
89  clickEnabled?: boolean;
90  config?: Record<string, any>;
91}

1// All 15 variable types
2type VariableType =
3  | 'text'
4  | 'number'
5  | 'boolean'
6  | 'timer'
7  | 'counter'
8  | 'date'
9  | 'state'
10  | 'list'
11  | 'object'
12  | 'map'
13  | 'set'
14  | 'json'
15  | 'computed'
16  | 'accumulated'
17  | 'mapped';
18
19// Base variable structure
20interface VariableBase {
21  id: string;
22  name: string;
23  type: VariableType;
24  description?: string;
25  category?: string;
26  hidden?: boolean;
27  onChange?: Rule[];
28}
29
30// Text variable
31interface TextVariable extends VariableBase {
32  type: 'text';
33  initialValue: string;
34  validation?: {
35    minLength?: number;
36    maxLength?: number;
37    pattern?: string;
38  };
39}
40
41// Number variable
42interface NumberVariable extends VariableBase {
43  type: 'number';
44  initialValue: number;
45  min?: number;
46  max?: number;
47  step?: number;
48}
49
50// Boolean variable
51interface BooleanVariable extends VariableBase {
52  type: 'boolean';
53  initialValue: boolean;
54}
55
56// Timer variable
57interface TimerVariable extends VariableBase {
58  type: 'timer';
59  initialValue: number;             // Starting value in ms
60  direction: 'up' | 'down';
61  target?: number;                  // Target value for conditions
62  autoStart?: boolean;
63  pauseWithVideo?: boolean;
64}
65
66// Counter variable
67interface CounterVariable extends VariableBase {
68  type: 'counter';
69  initialValue: number;
70  min?: number;
71  max?: number;
72  step?: number;
73  overflow?: 'clamp' | 'wrap' | 'error';
74}
75
76// State machine variable
77interface StateVariable extends VariableBase {
78  type: 'state';
79  initialState: string;
80  states: string[];
81  transitions: StateTransition[];
82}
83
84interface StateTransition {
85  from: string;
86  to: string;
87  condition?: Condition;
88}
89
90// Date variable
91interface DateVariable extends VariableBase {
92  type: 'date';
93  initialValue: string | number | null;
94  format?: string;
95  timezone?: string;                    // IANA timezone identifier
96}
97
98// List/Array variable
99interface ListVariable extends VariableBase {
100  type: 'list';
101  initialValue: any[];
102  itemType?: 'string' | 'number' | 'object';
103  maxLength?: number;
104  unique?: boolean;
105}
106
107// Object variable
108interface ObjectVariable extends VariableBase {
109  type: 'object';
110  initialValue: Record<string, any>;
111  schema: {
112    fields: Array<{
113      key: string;
114      type: 'string' | 'number' | 'boolean';
115      required?: boolean;
116      default?: any;
117      label?: string;
118    }>;
119  };
120}
121
122// Map variable
123interface MapVariable extends VariableBase {
124  type: 'map';
125  initialValue: Record<string, any>;
126  keyType: 'string';
127  valueType: 'string' | 'number' | 'boolean';
128}
129
130// Set variable
131interface SetVariable extends VariableBase {
132  type: 'set';
133  initialValue: any[];
134  itemType: 'string' | 'number';
135  maxSize?: number;
136}
137
138// JSON variable
139interface JsonVariable extends VariableBase {
140  type: 'json';
141  initialValue: any;
142}
143
144// Computed variable
145interface ComputedVariable extends VariableBase {
146  type: 'computed';
147  expression: string;
148  dependencies: string[];
149  resultType?: 'string' | 'number' | 'boolean';
150  fallback?: any;
151}
152
153// Accumulated variable
154interface AccumulatedVariable extends VariableBase {
155  type: 'accumulated';
156  sourceVariable: string;
157  aggregation: 'sum' | 'avg' | 'min' | 'max' | 'count';
158  windowSize?: number;
159  sampleRate?: number;
160}
161
162// Mapped variable
163interface MappedVariable extends VariableBase {
164  type: 'mapped';
165  sourceVariable: string;
166  segments: MappedSegment[];
167  defaultOutput?: any;
168  easing?: 'linear' | 'easeIn' | 'easeOut' | 'easeInOut';
169}
170
171interface MappedSegment {
172  from: number;
173  to: number;
174  output: any;
175}
176
177// Union type
178type Variable =
179  | TextVariable
180  | NumberVariable
181  | BooleanVariable
182  | TimerVariable
183  | CounterVariable
184  | DateVariable
185  | StateVariable
186  | ListVariable
187  | ObjectVariable
188  | MapVariable
189  | SetVariable
190  | JsonVariable
191  | ComputedVariable
192  | AccumulatedVariable
193  | MappedVariable;

1// Unified rule system
2interface UnifiedRule {
3  id: string;
4  name?: string;
5  enabled: boolean;
6  priority: number;                 // Higher = evaluated first
7
8  trigger?: UnifiedTrigger;
9  conditions: RuleGroup | Condition;
10  onMatch?: Action[];
11  onUnmatch?: Action[];
12
13  evaluateOn?: ('frame' | 'event' | 'stateChange')[];
14  cacheResult?: boolean;
15  cacheTTL?: number;
16  description?: string;
17}
18
19// Condition group
20interface RuleGroup {
21  logic: 'ALL' | 'ANY' | 'NONE';
22  conditions: (RuleGroup | Condition)[];
23}
24
25// All 21 condition types
26type ConditionType =
27  | 'variable'
28  | 'timerVariable'
29  | 'counterVariable'
30  | 'stateMachineVariable'
31  | 'arrayVariable'
32  | 'state'
33  | 'event'
34  | 'eventData'
35  | 'elementStats'
36  | 'playbackStatus'
37  | 'time'
38  | 'frame'
39  | 'object'
40  | 'userAttribute'
41  | 'abTest'
42  | 'device'
43  | 'geo'
44  | 'schedule'
45  | 'pluginState'
46  | 'expression'
47  | 'inputKey';
48
49// Comparison operators
50type ComparisonOperator =
51  | 'eq' | 'neq'
52  | 'gt' | 'gte' | 'lt' | 'lte'
53  | 'in' | 'notIn'
54  | 'contains' | 'startsWith' | 'endsWith'
55  | 'matches'
56  | 'exists' | 'notExists'
57  | 'between';
58
59// Base condition
60interface ConditionBase {
61  type: ConditionType;
62  operator?: ComparisonOperator;
63  value?: any;
64  value2?: any;                     // For 'between' operator
65}
66
67// Variable condition
68interface VariableCondition extends ConditionBase {
69  type: 'variable';
70  name: string;
71}
72
73// Time condition
74interface TimeCondition extends ConditionBase {
75  type: 'time';
76  timeType: 'playback' | 'wallClock' | 'sessionDuration';
77}
78
79// Object condition
80interface ObjectCondition extends ConditionBase {
81  type: 'object';
82  objectId?: string;
83  groupId?: string;
84  check: 'isVisible' | 'anyVisible' | 'confidence';
85}
86
87// Device condition
88interface DeviceCondition extends ConditionBase {
89  type: 'device';
90  deviceType: 'mobile' | 'tablet' | 'desktop';
91}
92
93// Element stats condition
94interface ElementStatsCondition extends ConditionBase {
95  type: 'elementStats';
96  elementId: string;
97  stat: 'hoverCount' | 'clickCount' | 'dwellTimeMs' | 'firstInteractionAt' | 'lastInteractionAt';
98}
99
100// Playback status condition
101interface PlaybackStatusCondition extends ConditionBase {
102  type: 'playbackStatus';
103  check: 'isPlaying' | 'isPaused' | 'isMuted' | 'volume' | 'playbackRate';
104}
105
106// Expression condition
107interface ExpressionCondition extends ConditionBase {
108  type: 'expression';
109  expression: string;
110}
111
112// Union type (simplified - add more as needed)
113type Condition =
114  | VariableCondition
115  | TimeCondition
116  | ObjectCondition
117  | DeviceCondition
118  | ElementStatsCondition
119  | PlaybackStatusCondition
120  | ExpressionCondition
121  | ConditionBase;

1// Overlay type
2interface Overlay {
3  id: string;
4  name: string;
5  type: string;                     // Plugin type (e.g., "core:button")
6  zIndex: number;
7  opacity: number;
8
9  // Positioning
10  position: OverlayPosition;
11  size?: OverlaySize;
12
13  // Visibility
14  visibility: OverlayVisibility;
15
16  // Plugin configuration
17  config: Record<string, any>;
18
19  // Object binding (for attached overlays)
20  objectBinding?: OverlayObjectBinding;
21
22  // Animation
23  animation?: OverlayAnimation;
24
25  // Events
26  eventHandlers?: Record<string, Action[]>;
27  eventRules?: Record<string, Rule[]>;
28}
29
30interface OverlayPosition {
31  type: 'fixed' | 'attached' | 'absolute';
32
33  // For fixed/absolute
34  x?: number;
35  y?: number;
36  anchor?: 'top-left' | 'top-center' | 'top-right' |
37           'center-left' | 'center' | 'center-right' |
38           'bottom-left' | 'bottom-center' | 'bottom-right';
39
40  // For attached (to object)
41  objectId?: string;
42  offset?: { x: number; y: number };
43  followObject?: boolean;
44}
45
46interface OverlaySize {
47  width: number | 'auto';
48  height: number | 'auto';
49  minWidth?: number;
50  maxWidth?: number;
51  minHeight?: number;
52  maxHeight?: number;
53  aspectRatio?: number;
54  scaleWithVideo?: boolean;
55}
56
57interface OverlayVisibility {
58  fromFrame?: number;
59  toFrame?: number;
60  showWhen?: Condition | RuleGroup;
61  hideWhen?: Condition | RuleGroup;
62  rules?: Rule[];
63}
64
65interface OverlayObjectBinding {
66  objectId: string;
67  position: 'above' | 'below' | 'left' | 'right' | 'center';
68  offset?: { x: number; y: number };
69  showOnHover?: boolean;
70  showOnClick?: boolean;
71  followPosition?: boolean;
72}
73
74interface OverlayAnimation {
75  enter?: AnimationConfig;
76  exit?: AnimationConfig;
77  idle?: AnimationConfig;
78}
79
80interface AnimationConfig {
81  type: 'fade' | 'slide' | 'scale' | 'bounce' | 'none';
82  duration?: number;
83  delay?: number;
84  easing?: string;
85  direction?: 'up' | 'down' | 'left' | 'right';
86}
87
88// Layer structure
89interface Layer {
90  id: string;
91  name: string;
92  order: number;
93  enabled: boolean;
94  blendMode: 'merge' | 'replace' | 'exclusive';
95
96  overlays: Overlay[];
97  rules?: LayerRule;
98  visibilityRules?: LayerVisibilityRule;
99
100  locked?: boolean;
101  color?: string;
102  collapsed?: boolean;
103}
104
105interface LayerVisibilityRule {
106  logic: 'ALL' | 'ANY' | 'NONE';
107  conditions: (RuleGroup | Condition)[];
108}

1// Common action types (69 total — see Actions Reference for complete list)
2type ActionType =
3  | 'seek'
4  | 'play'
5  | 'pause'
6  | 'goToScene'
7  | 'openUrl'
8  | 'navigate'
9  | 'showOverlay'
10  | 'hideOverlay'
11  | 'toggleOverlay'
12  | 'setVariable'
13  | 'setState'
14  | 'timerControl'
15  | 'counterControl'
16  | 'arrayControl'
17  | 'trackEvent'
18  | 'emitEvent'
19  | 'pluginAction'
20  | 'showNotification'
21  | 'showTracking'
22  | 'hideTracking'
23  | 'setPlaybackRate'
24  | 'setVolume'
25  | 'mute'
26  | 'unmute'
27  | 'wait'
28  | 'showCustomModal'
29  | 'closeCustomModal'
30  | 'setOverlayStyle'
31  | string;  // 69 types total — extensible
32
33// Base action
34interface ActionBase {
35  type: ActionType;
36  delay?: number;                   // Delay before execution (ms)
37  condition?: Condition;            // Only execute if condition met
38}
39
40// Seek action
41interface SeekAction extends ActionBase {
42  type: 'seek';
43  time?: number;                    // Seconds
44  frame?: number;                   // Frame number
45  relative?: boolean;               // Relative to current position
46}
47
48// Open URL action
49interface OpenUrlAction extends ActionBase {
50  type: 'openUrl';
51  url: string;
52  target?: 'blank' | 'self' | 'parent' | 'top';
53}
54
55// Overlay actions
56interface ShowOverlayAction extends ActionBase {
57  type: 'showOverlay';
58  overlayId: string;
59}
60
61interface HideOverlayAction extends ActionBase {
62  type: 'hideOverlay';
63  overlayId: string;
64}
65
66// Playback actions
67interface PauseAction extends ActionBase {
68  type: 'pause';
69  reason?: string;
70}
71
72interface PlayAction extends ActionBase {
73  type: 'play';
74}
75
76interface SetPlaybackRateAction extends ActionBase {
77  type: 'setPlaybackRate';
78  rate: number;                     // 0.25 - 2.0
79}
80
81interface SetVolumeAction extends ActionBase {
82  type: 'setVolume';
83  volume: number;                   // 0.0 - 1.0
84}
85
86// State action
87interface SetStateAction extends ActionBase {
88  type: 'setState';
89  key: string;
90  value: any;                       // Can be expression string
91}
92
93// Track event action
94interface TrackEventAction extends ActionBase {
95  type: 'trackEvent';
96  eventName: string;
97  properties?: Record<string, any>;
98}
99
100// Highlight object action
101interface HighlightObjectAction extends ActionBase {
102  type: 'highlightObject';
103  objectId: string;
104  duration?: number;                // ms
105  style?: {
106    color?: string;
107    borderWidth?: number;
108    animation?: string;
109  };
110}
111
112// Show notification action
113interface ShowNotificationAction extends ActionBase {
114  type: 'showNotification';
115  message: string;
116  variant?: 'info' | 'success' | 'warning' | 'error';
117  duration?: number;                // ms (0 = persistent)
118  position?: 'top' | 'bottom' | 'top-left' | 'top-right' | 'bottom-left' | 'bottom-right';
119}
120
121// Plugin action
122interface CallPluginAction extends ActionBase {
123  type: 'callPlugin';
124  plugin: string;                   // pluginName.methodName
125  params?: Record<string, any>;
126}
127
128// Union type
129type Action =
130  | SeekAction
131  | OpenUrlAction
132  | ShowOverlayAction
133  | HideOverlayAction
134  | PauseAction
135  | PlayAction
136  | SetPlaybackRateAction
137  | SetVolumeAction
138  | SetStateAction
139  | TrackEventAction
140  | HighlightObjectAction
141  | ShowNotificationAction
142  | CallPluginAction
143  | { type: 'mute' }
144  | { type: 'unmute' };

1// Player component props
2interface MP4EPlayerProps {
3  // Source
4  src: string;
5  metadata?: MP4EMetadata | string;
6
7  // Playback
8  autoplay?: boolean;
9  muted?: boolean;
10  loop?: boolean;
11  preload?: 'none' | 'metadata' | 'auto';
12  playbackRate?: number;
13  startTime?: number;
14
15  // Display
16  width?: number | string;
17  height?: number | string;
18  aspectRatio?: '16:9' | '4:3' | '1:1' | 'auto';
19  objectFit?: 'contain' | 'cover' | 'fill';
20  poster?: string;
21
22  // Features
23  interactive?: boolean;
24  showObjects?: boolean;
25  showOverlays?: boolean;
26  showControls?: boolean;
27  objectGroups?: string[];
28
29  // Overrides
30  overrideVariables?: Record<string, any>;
31  overrideDisplaySettings?: DisplaySettings;
32
33  // Debug
34  debug?: boolean;
35  showDebugOverlay?: boolean;
36
37  // Callbacks - Lifecycle
38  onReady?: () => void;
39  onLoad?: (event: { duration: number }) => void;
40  onMetadataLoaded?: (metadata: MP4EMetadata) => void;
41  onEngineReady?: (engine: MP4EEngine) => void;
42  onDestroy?: () => void;
43
44  // Callbacks - Playback
45  onPlay?: () => void;
46  onPause?: (event: { currentTime: number; reason?: string }) => void;
47  onEnded?: () => void;
48  onTimeUpdate?: (event: { currentTime: number; currentFrame: number }) => void;
49  onSeeking?: (event: { fromTime: number; toTime: number }) => void;
50  onSeeked?: (event: { currentTime: number }) => void;
51  onBufferStart?: () => void;
52  onBufferEnd?: () => void;
53  onVolumeChange?: (event: { volume: number }) => void;
54  onRateChange?: (event: { playbackRate: number }) => void;
55  onFirstPlay?: () => void;
56
57  // Callbacks - Objects
58  onObjectClick?: (event: ObjectEvent) => void;
59  onObjectHover?: (event: ObjectEvent) => void;
60  onObjectHoverEnd?: (event: ObjectEvent) => void;
61  onObjectVisible?: (event: ObjectEvent) => void;
62  onObjectHidden?: (event: ObjectEvent) => void;
63
64  // Callbacks - Overlays
65  onOverlayShow?: (event: { overlayId: string }) => void;
66  onOverlayHide?: (event: { overlayId: string }) => void;
67  onOverlayClick?: (event: { overlayId: string }) => boolean | void;
68  onPluginEvent?: (event: { pluginId: string; eventName: string; data: any }) => void;
69
70  // Callbacks - Variables
71  onVariableChange?: (event: { variableName: string; previousValue: any; newValue: any }) => void;
72
73  // Callbacks - Actions
74  onAction?: (action: Action) => boolean | void;
75  onTrackEvent?: (event: { eventName: string; properties: Record<string, any> }) => void;
76
77  // Callbacks - Errors
78  onError?: (error: { message: string; code: string; details?: any }) => void;
79
80  // Styling
81  className?: string;
82  style?: React.CSSProperties;
83}
84
85interface ObjectEvent {
86  objectId: string;
87  objectLabel: string;
88  confidence: number;
89  boundingBox: { x: number; y: number; width: number; height: number };
90  objectData?: ObjectData;
91  clickX?: number;
92  clickY?: number;
93}
94
95// Player ref methods
96interface MP4EPlayerRef {
97  // Playback control
98  play(): void;
99  pause(): void;
100  seek(time: number): void;
101  seekToFrame(frame: number): void;
102  setPlaybackRate(rate: number): void;
103  setVolume(volume: number): void;
104  mute(): void;
105  unmute(): void;
106
107  // State getters
108  getCurrentTime(): number;
109  getCurrentFrame(): number;
110  getDuration(): number;
111  getTotalFrames(): number;
112  isPaused(): boolean;
113  isMuted(): boolean;
114  getVolume(): number;
115  getPlaybackRate(): number;
116
117  // Objects
118  getVisibleObjects(): ObjectData[];
119  getAllObjects(): ObjectRegistry;
120  highlightObject(objectId: string, duration?: number): void;
121
122  // Variables
123  getVariable(name: string): any;
124  setVariable(name: string, value: any): void;
125  resetVariables(): void;
126
127  // Actions
128  executeAction(action: Action): void;
129  executeActions(actions: Action[]): void;
130
131  // Overlays
132  showOverlay(overlayId: string): void;
133  hideOverlay(overlayId: string): void;
134
135  // Scenes
136  goToScene(sceneId: string): void;
137  getCurrentScene(): string | null;
138
139  // Engine access
140  getEngine(): MP4EEngine | null;
141  getMetadata(): MP4EMetadata | null;
142
143  // Video element
144  getVideoElement(): HTMLVideoElement | null;
145}