Design TikTok (Short Video Platform)

two_tower_retrieval_and_ranking_funnel:
  stage_1_candidate_retrieval:
    model: "Two Tower Dual Encoder"
    corpus_size: "10,000,000 active videos"
    candidate_output: "10,000 video candidates"
    latency_budget: "30 milliseconds"
    user_tower:
      inputs: "Demographics, long term interest vectors, historical completion rates, session language"
      output: "128-dimensional user embedding vector computed once per session and refreshed on interactions"
    video_tower:
      inputs: "Multimodal video features (audio spectrogram, visual keyframes, title text, hashtags)"
      output: "128-dimensional video embedding vector precomputed offline during video ingestion"
    vector_indexing: "Hierarchical Navigable Small World (HNSW) graph index in vector database"
    scoring_metric: "Cosine similarity / dot product: dot_product(user_vector, video_vector)"

  stage_2_deep_ranking:
    model: "Single Tower Neural Ranker (Mixture of Experts)"
    candidate_input: "10,000 candidates from Stage 1"
    lightweight_pre_rank: "Reduce 10,000 candidates to 1,000 candidates before expensive neural inference"
    neural_inference_input: "1,000 candidates"
    ranked_output: "500 ranked videos"
    latency_budget: "50 milliseconds"
    features: "Dense cross features combining [user_features, video_features, interaction_history, context_features]"
    prediction_targets:
      - "P(complete): Probability of watching full video duration"
      - "P(rewatch): Probability of looping the video a second or third time"
      - "P(share): Probability of sharing video externally or to friends"
      - "P(comment): Probability of leaving a comment"
      - "P(like): Probability of tapping the heart icon"
      - "P(skip): Probability of quickly swiping away"
    utility_score_formula: "Score = w1*P(complete) + w2*P(rewatch) + w3*P(share) + w4*P(comment) + w5*P(like) - w6*P(skip)"

  stage_3_business_rules_and_diversity:
    candidate_input: "Top 500 ranked videos"
    final_output: "200 curated video IDs stored in Redis feed buffer"
    latency_budget: "10 milliseconds"
    filtering_rules:
      - "Category diversity: Maximum 2 consecutive videos from the same creator or hashtag theme"
      - "Freshness exploration: Epsilon-greedy 10% slot reservation for newly uploaded videos"
      - "Author fatigue penalty: Reduce ranking weight for creators viewed within the last 2 hours"
      - "Safety verification: Exclude videos undergoing post publish report investigation"

precomputed_feed_buffering:
  storage_topology:
    feed_buffer_key: "feed:{user_id}"
    data_structure: "Redis List"
    buffer_capacity: "200 precomputed video IDs"
    ttl: "2 hours"
    read_latency: "< 50 milliseconds target via LRANGE feed:{user_id} 0 19"
    request_reranking: "After loading cached IDs, apply lightweight session aware reranking using recent completions, skips, and seen state before returning up to 20 videos"

  cache_lifecycle_and_invalidation:
    background_worker_cadence: "Evaluates active users every 30 minutes to replenish candidate buffers"
    active_user_optimization: "Recompute only for users with an active session plus a low buffer, significant feature change, or scheduled refresh opportunity"
    idle_user_fallback: "Users inactive for > 1 hour use their retained buffer while it exists. If it has expired, fall back to regional trending candidates while refreshing personalization"

  client_prefetching_strategy:
    initial_load: "Client requests 20 videos on startup, immediately playing video 1 while buffering 2 and 3"
    pipeline_prefetch: "As viewer watches video N, mobile player prefetches segments for videos N+1, N+2, and N+3"
    replenishment_trigger: "When the viewer reaches item 150 in the 200-item buffer, the serving layer schedules asynchronous feed replenishment"

engagement_signal_hierarchy:
  signal_1_watch_completion_rate:
    importance: "Primary ranking signal (approximately 10x more predictive of user delight than likes)"
    formula: "completion_rate = min(watch_time, video_duration) / video_duration"
    replay_signal: "watch_time_ratio = watch_time / video_duration. Values above 1.0 indicate replay consumption"
    significance:
      single_loop: "completion_rate = 1.0 indicates full narrative consumption"
      multi_loop: "watch_time_ratio > 1.0 (2nd or 3rd replay loop) represents the strongest positive affinity"

  signal_2_share:
    importance: "High-intent explicit social recommendation"
    significance: "Viewer advocates content to external peers, indicating exceptional quality"

  signal_3_comment:
    importance: "Strong positive engagement signal"
    significance: "Viewer invests active time writing feedback, driving community interaction"

  signal_4_like:
    importance: "Moderate positive signal"
    significance: "Low cognitive friction tap, with high volume but relatively noisy compared to watch time"

  signal_5_follow_after_watching:
    importance: "High-value creator loyalty signal"
    significance: "Directly converts viewer interest into sustained following relationship"

  signal_6_skip_or_swipe_away:
    importance: "Strong negative implicit signal"
    significance: "Swiping away in under 3 seconds penalizes video score and related feature weights"

  signal_7_explicit_not_interested:
    importance: "Strongest negative explicit signal"
    significance: "Explicitly tapping 'Not Interested' heavily suppresses the creator, audio track, and topic"

training_data_and_feedback_loop:
  session_events: "Viewer interactions generate real time training tuples: (user_features, video_features) -> {completed, liked, shared, skipped}"
  model_retraining: "Models undergo daily batch retraining on distributed Spark/GPU clusters and hourly streaming fine-tuning so viral trends can influence models within hours"

content_moderation_pipeline:
  stage_1_automated_classifiers:
    execution_time: "< 5 seconds per uploaded video"
    image_analysis: "Convolutional and Vision Transformer models scanning keyframes for nudity, violence, and gore"
    text_analysis: "NLP transformer models evaluating captions, subtitles, and OCR video text for hate speech"
    audio_analysis: "Acoustic fingerprint matching against copyrighted sound and music databases"
    metadata_heuristics: "Pattern detection identifying spam, scam links, and coordinated bot behaviors"
    output: "Normalized confidence score between 0.0 and 1.0 across each violation category"

  stage_2_decision_routing:
    auto_reject:
      condition: "Confidence score > 0.95 in any severe policy category"
      action: "Block video publication immediately, log audit record, and notify creator with policy citation"
    human_review_queue:
      condition: "Confidence score between 0.70 and 0.95"
      action: "Hold video in processing state, excluding from public feeds pending human operator inspection"
    auto_approve:
      condition: "Confidence score < 0.70 across all policy categories"
      action: "Mark the video moderation-eligible. Publication still waits for required renditions and the authoritative publish transaction"

  stage_3_human_review_operations:
    queue_volume: "10,000 to 50,000 borderline videos per day routed to human moderation"
    operational_sla: "Complete review within 2 hours of upload"
    staffing_hierarchy: "Trained moderation operations with escalation pathways to specialized legal and policy teams"

  stage_4_post_publish_monitoring:
    signal_source: "Live user reports and anomaly detection algorithms tracking viewer flag spikes"
    deprioritize_threshold:
      condition: "Accumulates more than 10 user reports"
      action: "Automatically suppress recommendation weight and remove from exploratory FYP slots"
    takedown_threshold:
      condition: "Accumulates more than 50 user reports"
      action: "Instantly remove from all discovery feeds and enqueue high-priority human review"

  accuracy_benchmarks:
    appeals_process: "Creators can appeal removals, triggering an independent secondary human review"
    target_false_positive_rate: "< 1.0% (legitimate videos mistakenly removed)"
    target_false_negative_rate: "< 0.1% (violating media mistakenly exposed to viewers)"

kafka_cluster_topology:
  video_lifecycle_topic:
    topic_name: "video-lifecycle"
    partition_key: "video_id"
    partition_count: 256
    retention_period: "7 days"
    replication_factor: 3
    purpose: "Carries upload-ready, published, and removed lifecycle events while preserving per-video ordering"

  engagement_events_topic:
    topic_name: "engagement-events"
    partition_key: "user_id"
    partition_count: 256
    retention_period: "7 days"
    replication_factor: 3
    purpose: "Streams viewer telemetry plus asynchronously emitted engagement events after authoritative mutation commits"

  social_events_topic:
    topic_name: "social-events"
    partition_key: "actor_user_id"
    partition_count: 128
    retention_period: "7 days"
    replication_factor: 3
    purpose: "Streams follows, mentions, and other social mutations used by notifications and relationship features"

event_schemas:
  video_lifecycle_event:
    event_id: "CHAR(36) unique event identifier"
    event_type: "ENUM(upload_ready, published, removed)"
    video_id: "VARCHAR(36) UUID"
    creator_id: "BIGINT creator identifier"
    s3_key: "raw/uploads/2026/03/video-uuid.mp4"
    duration_sec: "INTEGER (video length in seconds)"
    timestamp: "TIMESTAMP (epoch milliseconds)"

  engagement_event:
    event_id: "UUID event transaction ID"
    user_id: "UUID viewer identifier"
    creator_id: "UUID creator identifier when applicable"
    video_id: "UUID video identifier"
    event_kind: "ENUM(telemetry, mutation_event)"
    action: "ENUM(watch_start, heartbeat, like, share, comment, skip, not_interested)"
    watch_ms: "INTEGER (playback duration in milliseconds)"
    country: "STRING country or region code"
    device_type: "STRING client device category"
    timestamp: "TIMESTAMP (epoch milliseconds)"

  social_event:
    event_id: "UUID event transaction ID"
    actor_user_id: "UUID acting user identifier"
    target_user_id: "UUID affected user identifier"
    video_id: "UUID video identifier when applicable"
    action: "ENUM(follow, mention)"
    timestamp: "TIMESTAMP (epoch milliseconds)"

consumer_groups:
  transcoding_workers:
    purpose: "Consumes upload_ready lifecycle events to execute GPU transcoding, thumbnail generation, and initial moderation processing"
  rec_feature_updater:
    purpose: "Apache Flink job consuming engagement-events to update Redis user and video feature vectors in real time and repartitioning by video_id where video-level aggregates are required"
  vector_index_updater:
    purpose: "Consumes published and removed lifecycle events, upserts eligible 128-dimensional video embeddings into the HNSW vector index, and removes or tombstones deleted videos"
  search_index_updater:
    purpose: "Indexes published video metadata, hashtags, creator handles, captions, and audio identifiers, then removes or tombstones deleted videos"
  model_training_pipeline:
    purpose: "Apache Spark ETL job dumping events to S3 Parquet for offline deep neural network retraining"
  creator_analytics_pipeline:
    purpose: "Aggregates engagement and social events for near real time creator dashboards and daily reconciliation in ClickHouse"
  trending_pipeline:
    purpose: "Apache Flink job computes five minute velocity and acceleration for hashtags, sounds, and topic categories, then writes regional top K trends to Redis for discovery and viral notification triggers"
  notification_pipeline:
    purpose: "Consumes committed likes, comments, and social mutations, resolves recipients from creator_id or target user IDs, applies notification preferences and deduplication, rate limits bursts, and dispatches through APNs or FCM with retry and dead letter handling"

operational_paths:
  upload_and_publish_path: "Client requests a presigned URL and uploads to S3. The publish API verifies that the object exists and records video metadata plus an upload_ready lifecycle outbox event in one MySQL transaction. Debezium CDC publishes the committed outbox event to Kafka after commit"
  publication_transition: "After required renditions and safety checks succeed, a transaction sets status = published and writes a published lifecycle outbox event. A removal transaction sets status = removed and writes a removed lifecycle event. Conditional state transitions prevent duplicate publish or remove events"
  feed_telemetry_path: "Client emits engagement heartbeats that flow into recommendation pipelines without blocking playback. Authoritative likes, comments, and follows are persisted through their mutation APIs and publish mutation events only after commit"
  notification_path: "Committed like, comment, follow, and mention events resolve the recipient, check notification preferences, deduplicate and rate limit the notification, then enqueue APNs or FCM delivery. Delivery failures retry with backoff before entering a dead letter queue"
  trending_path: "Flink aggregates five minute regional velocity and acceleration for hashtags, sounds, and categories. Top K results are written to Redis, and threshold crossings can emit viral trend notification events"
  dead_letter_queue: "Failed engagement events route to engagement-events-dlq, while failed lifecycle events route to video-lifecycle-dlq. Alert when the relevant consumer lag exceeds 5 minutes"
  idempotency: "Consumers use event_id to deduplicate retried CDC or Kafka deliveries"