Plugin API Reference

// The 'object' global is available when plugin is triggered by an object click/hover
{
  id: "obj_watch_1",           // Unique object identifier
  label: "watch",              // Original AI-detected label
  userLabel: "Apple Watch",    // User-customized label (optional)
  position: {
    x: 320,                    // Center X position
    y: 180,                    // Center Y position
    width: 150,                // Bounding box width
    height: 100,               // Bounding box height
    bbox: [245, 130, 395, 230] // [x1, y1, x2, y2]
  },
  data: {                      // Custom data from metadata
    price: 399,
    sku: "WATCH-001",
    description: "Series 9 GPS",
    productUrl: "https://example.com/watch"
  }
}

// Access plugin configuration values
const config = mp4e.getConfig();

// config contains values from the plugin editor:
{
  title: "Product Details",
  showPrice: true,
  currency: "USD",
  buttonText: "Add to Cart",
  // ... any values defined in your plugin settings
}

// Use in your code:
document.getElementById('title').textContent = config.title;
if (config.showPrice) {
  document.getElementById('price').textContent =
    config.currency + ' ' + object.data.price;
}

// Access the full data object
const data = mp4e.getData();

// data structure:
{
  object: { ... },        // Same as the 'object' global
  groups: [               // Object groups this object belongs to
    { id: "products", name: "Products", color: "#3b82f6" },
    { id: "featured", name: "Featured Items", color: "#10b981" }
  ]
}

// GET a variable (async - returns Promise)
const cartCount = await mp4e.getVariable('cartCount');
const userName = await mp4e.getVariable('userName');

// SET a variable (async - returns Promise)
await mp4e.setVariable('selectedProduct', object.label);
await mp4e.setVariable('cartCount', cartCount + 1);

// Multiple operations
const [price, quantity] = await Promise.all([
  mp4e.getVariable('unitPrice'),
  mp4e.getVariable('quantity')
]);
const total = price * quantity;
await mp4e.setVariable('total', total);

// Get current playback position (synchronous - updated via bridge)
const currentTime = mp4e.getCurrentTime();  // seconds
const currentFrame = mp4e.getCurrentFrame(); // frame number

// Control playback (fire-and-forget)
mp4e.playVideo();
mp4e.pauseVideo();
mp4e.seekToTime(30);       // Seek to 30 seconds
mp4e.goToScene('chapter-2'); // Jump to named scene

// Volume and rate control
mp4e.setVolume(0.5);          // Set volume to 50%
mp4e.setPlaybackRate(1.5);    // Set playback speed to 1.5x

// Query playback state (synchronous)
const duration = mp4e.getDuration();         // Total duration in seconds
const playing = mp4e.isPlaying();            // true if currently playing
const buffering = mp4e.isBuffering();        // true if buffering
const volume = mp4e.getVolume();             // Current volume (0-1)
const muted = mp4e.isMuted();               // true if muted
const rate = mp4e.getPlaybackRate();         // Current playback rate
const fullscreen = mp4e.isFullscreen();      // true if in fullscreen

// Subscribe to keyboard events forwarded from the parent
mp4e.on('input:keydown', (data) => {
  if (data.key === 'Escape') {
    mp4e.close();
  }
});

// Listen for config updates (when variables change config values)
mp4e.onConfigUpdate((newConfig) => {
  document.getElementById('title').textContent = newConfig.title;
});

// Listen for variable changes
mp4e.onVariableChange((variables, changedIds) => {
  // changedIds is an array of variable IDs that changed,
  // or undefined for full broadcast (e.g., initial load)
  if (changedIds && !changedIds.includes('score')) return;
  document.getElementById('score').textContent = variables.score;
});

// Unsubscribe from a named event
mp4e.off('input:keydown', myHandler);

// Fade in with scale
mp4e.setStyle(
  { opacity: 1, scale: 1.05 },
  { durationMs: 300, easing: 'ease-out' }
);

// Reset after delay
setTimeout(() => mp4e.resetStyle(), 2000);

// Apply style without animation
mp4e.setStyle({ borderRadius: '50%', rotation: 45 });

// Combine with wait for timed effects
mp4e.executeActions([
  { type: 'setOverlayStyle', overlayId: 'my-overlay',
    scale: 1.5, durationMs: 300, easing: 'ease-out' },
  { type: 'wait', durationMs: 2000 },
  { type: 'resetOverlayStyle', overlayId: 'my-overlay' }
]);

// Available style properties:
mp4e.setStyle({
  opacity: 0.5,              // 0-1 opacity
  scale: 1.2,                // Scale factor
  rotation: 45,              // Degrees
  borderRadius: '12px',      // CSS border-radius
  boxShadow: '0 4px 12px rgba(0,0,0,0.3)',
  backgroundColor: '#ff000080',
  translateX: '10px',        // Horizontal offset
  translateY: '-5px',        // Vertical offset
  filter: 'blur(2px)',       // CSS filter
  backdropFilter: 'blur(8px)', // CSS backdrop-filter
}, {
  durationMs: 500,           // Animation duration (ms)
  easing: 'ease-out',        // CSS easing function
  delayMs: 100               // Animation delay (ms)
});

// Get current viewport dimensions
const viewport = mp4e.getViewport();
// { width: 1280, height: 720, scaleFactor: 0.75, hasTouch: false, orientation: 'landscape' }

// Respond to viewport changes (resize, orientation, scale)
mp4e.onViewportChange((viewport) => {
  if (viewport.orientation === 'portrait') {
    document.body.classList.add('mobile-layout');
  } else {
    document.body.classList.remove('mobile-layout');
  }
});

// Later, unsubscribe
mp4e.offViewportChange(myCallback);

// Resolve a media:// URL to a usable blob URL
const blobUrl = await mp4e.getMediaUrl('media://product-image-001');
document.getElementById('img').src = blobUrl;

// Download an embedded asset
await mp4e.downloadMedia('media://brochure-pdf', 'brochure.pdf');

// List all embedded media
const items = await mp4e.listMedia();
// [{ id: 'product-image-001', type: 'image/png', purpose: 'asset', metadata: {...} }, ...]

// Get metadata for a specific item without loading the binary
const info = await mp4e.statMedia('product-image-001');
// { id: 'product-image-001', type: 'image/png', purpose: 'asset', metadata: { width: 800, height: 600 } }

// Store user preferences
await mp4e.storage.set('theme', 'dark');
await mp4e.storage.set('fontSize', 14);

// Retrieve stored values
const theme = await mp4e.storage.get('theme');   // 'dark'
const size = await mp4e.storage.get('fontSize');  // 14

// List all keys
const keys = await mp4e.storage.keys();  // ['theme', 'fontSize']

// Remove a single key
await mp4e.storage.remove('theme');

// Clear all plugin storage
await mp4e.storage.clear();

// GET request
const response = await mp4e.fetch('https://api.example.com/products');
if (response.ok) {
  const products = response.data;
  console.log('Products:', products);
}

// POST request with body
const result = await mp4e.fetch('https://api.example.com/cart', {
  method: 'POST',
  headers: { 'Content-Type': 'application/json' },
  body: JSON.stringify({ productId: 'prod_123', quantity: 1 })
});

if (result.ok) {
  mp4e.executeActions([
    { type: 'showNotification', message: 'Added to cart!', type: 'success' }
  ]);
}

// Shopping cart plugin registers action handler
mp4e.onAction('addItem', (config) => {
  cart.push({ name: config.productName, price: config.price });
  mp4e.setVariable('cart:itemCount', cart.length);
  return { success: true };
});

// Product card plugin calls cart action
const result = await mp4e.callAction('cart', 'addItem', {
  productName: 'Blue T-Shirt',
  price: 29.99
});

// Execute one or more actions
mp4e.executeActions([
  // Playback actions
  { type: 'pause' },
  { type: 'seek', time: 45 },
  { type: 'setPlaybackRate', rate: 1.5 },
  { type: 'setVolume', volume: 0.8 },

  // Navigation
  { type: 'openUrl', url: 'https://example.com', target: '_blank' },
  { type: 'goToScene', sceneId: 'ending' },

  // UI actions
  { type: 'showOverlay', overlayId: 'promo-banner' },
  { type: 'hideOverlay', overlayId: 'intro-tooltip' },
  { type: 'showNotification', message: 'Added to cart!', type: 'success' },
  { type: 'enterFullscreen' },

  // Variable actions
  { type: 'setVariable', name: 'cartCount', value: 5 },
  { type: 'setVariable', name: 'total', operation: 'add', value: 99.99 },

  // Custom plugin actions
  {
    type: 'showCustomModal',
    pluginId: 'plugin_checkout_form',
    config: {
      title: 'Complete Purchase',
      productName: object.label,
      price: object.data.price
    },
    pauseVideo: true
  },

  // Analytics
  { type: 'trackEvent', event: 'add_to_cart', data: { product: object.id } },

  // API calls
  {
    type: 'api',
    url: 'https://api.example.com/cart',
    method: 'POST',
    body: { productId: object.data.sku },
    storeResultIn: 'apiResponse'
  }
]);

// Control video playback
mp4e.executeActions([
  { type: 'play' },                    // Start playback
  { type: 'pause' },                   // Pause playback
  { type: 'seek', time: 30 },          // Seek to 30 seconds
  { type: 'seekRelative', offset: -5 }, // Seek back 5 seconds
  { type: 'setPlaybackRate', rate: 2 }, // 2x speed
  { type: 'setVolume', volume: 0.5 },  // 50% volume
  { type: 'mute' },                     // Mute audio
  { type: 'unmute' },                   // Unmute audio
  { type: 'setPlaybackStart', time: 10 }, // Set start point
  { type: 'setPlaybackEnd', time: 60 },   // Set end point
]);

// Navigation and URL handling
mp4e.executeActions([
  { type: 'goToScene', sceneId: 'chapter-3' },
  { type: 'openUrl', url: 'https://example.com', target: '_blank' },
  { type: 'openUrl', url: '/product/123', target: '_self' },
]);

// UI control
mp4e.executeActions([
  { type: 'showOverlay', overlayId: 'overlay_banner' },
  { type: 'hideOverlay', overlayId: 'overlay_intro' },
  { type: 'toggleOverlay', overlayId: 'overlay_menu' },
  { type: 'activateLayer', layerId: 'layer_products' },
  { type: 'deactivateLayer', layerId: 'layer_tutorial' },
  { type: 'highlightObject', objectId: 'obj_chair_1', duration: 3000 },
  { type: 'showNotification', message: 'Welcome!', type: 'info', duration: 5000 },
  { type: 'enterFullscreen' },
  { type: 'exitFullscreen' },
]);

// Variable manipulation
mp4e.executeActions([
  // Set value directly
  { type: 'setVariable', name: 'score', value: 100 },

  // Mathematical operations
  { type: 'setVariable', name: 'score', operation: 'add', value: 10 },
  { type: 'setVariable', name: 'score', operation: 'subtract', value: 5 },
  { type: 'setVariable', name: 'score', operation: 'multiply', value: 2 },
  { type: 'setVariable', name: 'score', operation: 'divide', value: 4 },

  // String operations
  { type: 'setVariable', name: 'message', operation: 'append', value: ' World' },

  // Timer control
  { type: 'timerControl', name: 'watchTimer', action: 'start' },
  { type: 'timerControl', name: 'watchTimer', action: 'pause' },
  { type: 'timerControl', name: 'watchTimer', action: 'reset' },

  // Array operations
  { type: 'arrayControl', name: 'cart', action: 'push', value: 'item-1' },
  { type: 'arrayControl', name: 'cart', action: 'pop' },
  { type: 'arrayControl', name: 'cart', action: 'remove', value: 'item-1' },
  { type: 'arrayControl', name: 'cart', action: 'clear' },

  // Custom events
  { type: 'emitEvent', event: 'product-selected', data: { id: 'prod_123' } },
]);

// Built-in dialogs
mp4e.executeActions([
  // Simple alert
  { type: 'alert', message: 'Video complete!' },

  // Confirmation dialog
  {
    type: 'confirm',
    message: 'Add to cart?',
    onConfirm: [
      { type: 'setVariable', name: 'cartCount', operation: 'add', value: 1 },
      { type: 'showNotification', message: 'Added!', type: 'success' }
    ],
    onCancel: [
      { type: 'showNotification', message: 'Cancelled', type: 'info' }
    ]
  },

  // Prompt for input
  {
    type: 'prompt',
    message: 'Enter quantity:',
    inputType: 'number',
    defaultValue: '1',
    storeResultIn: 'userQuantity'
  }
]);

// Custom plugin modal
mp4e.executeActions([
  {
    type: 'showCustomModal',
    pluginId: 'plugin_checkout',  // Your plugin ID from the editor
    config: {
      title: 'Complete Purchase',
      productName: object.label,
      productImage: object.data.imageUrl,
      price: object.data.price,
      currency: 'USD'
    },
    showBackdrop: true,
    pauseVideo: true   // Pause video while modal is open
  }
]);

// Custom overlay
mp4e.executeActions([
  {
    type: 'showCustomOverlay',
    pluginId: 'plugin_banner',
    position: { x: 50, y: 10 },  // Percentage position
    config: { text: 'Limited time offer!' }
  },
  { type: 'hideCustomOverlay', pluginId: 'plugin_banner' }
]);

// Analytics and API
mp4e.executeActions([
  // Track custom event
  {
    type: 'trackEvent',
    event: 'product_view',
    data: {
      productId: object.data.sku,
      productName: object.label,
      category: 'electronics'
    }
  },

  // Make API call
  {
    type: 'api',
    url: 'https://api.example.com/track',
    method: 'POST',
    headers: { 'Authorization': 'Bearer token123' },
    body: {
      event: 'video_interaction',
      objectId: object.id
    },
    storeResultIn: 'apiResponse'
  },

  // Debug logging
  { type: 'log', message: 'User clicked: ' + object.label },

  // Wait before next action
  { type: 'wait', durationMs: 1000 }  // 1 second
]);

1// Complete example: Interactive product card plugin
2
3// ===== HTML (in plugin editor) =====
4<div class="product-card">
5  <img id="product-image" src="{{object.data.imageUrl}}" />
6  <h3 id="product-name">{{object.userLabel}}</h3>
7  <p id="product-price"></p>
8  <button id="add-to-cart">Add to Cart</button>
9  <button id="buy-now">Buy Now</button>
10</div>
11
12// ===== CSS =====
13.product-card {
14  background: white;
15  border-radius: 12px;
16  padding: 16px;
17  box-shadow: 0 4px 12px rgba(0,0,0,0.15);
18}
19#product-image { width: 100%; border-radius: 8px; }
20button {
21  padding: 8px 16px;
22  border-radius: 6px;
23  cursor: pointer;
24  margin-top: 8px;
25}
26#add-to-cart { background: #3b82f6; color: white; }
27#buy-now { background: #10b981; color: white; }
28
29// ===== JavaScript =====
30(async function() {
31  // Get config and format price
32  const config = mp4e.getConfig();
33  const price = object.data?.price || 0;
34  const formatted = new Intl.NumberFormat('en-US', {
35    style: 'currency',
36    currency: config.currency || 'USD'
37  }).format(price);
38  document.getElementById('product-price').textContent = formatted;
39
40  // Add to Cart button
41  document.getElementById('add-to-cart').addEventListener('click', async () => {
42    // Get current cart count
43    const cartCount = await mp4e.getVariable('cartCount') || 0;
44
45    // Update cart
46    await mp4e.setVariable('selectedProduct', object.userLabel || object.label);
47    await mp4e.setVariable('selectedPrice', price);
48    await mp4e.setVariable('cartCount', cartCount + 1);
49
50    // Show notification
51    mp4e.executeActions([
52      { type: 'showNotification', message: 'Added to cart!', type: 'success' },
53      { type: 'trackEvent', event: 'add_to_cart', data: { product: object.id, price } }
54    ]);
55
56    // Close this plugin
57    mp4e.close();
58  });
59
60  // Buy Now button
61  document.getElementById('buy-now').addEventListener('click', () => {
62    mp4e.executeActions([
63      { type: 'pause' },
64      {
65        type: 'showCustomModal',
66        pluginId: 'plugin_checkout_form',
67        config: {
68          productName: object.userLabel || object.label,
69          productImage: object.data?.imageUrl,
70          price: price,
71          sku: object.data?.sku
72        },
73        pauseVideo: true,
74        showBackdrop: true
75      },
76      { type: 'trackEvent', event: 'buy_now_click', data: { product: object.id } }
77    ]);
78  });
79})();