Design Top K Most Shared Articles

pane_based_counting:
  time_slice_unit: "5 minute discrete panes"
  pane_storage: "Aggregated count of shares per article, region, and category within each 5 minute slice"
  window_aggregations:
    one_hour: "Sum of the most recent 12 panes"
    twenty_four_hours: "Sum of the most recent 288 panes"
    seven_days: "Sum of the most recent 2,016 panes"
    thirty_days: "Sum of the most recent 8,640 panes"
  memory_optimization:
    threshold_filter: "Heuristic: only articles with more than 10 shares in any pane enter candidate state. Validate this threshold against the 5% accuracy SLA because it can exclude low-velocity articles whose shares accumulate across panes"
    memory_calculation: "100K active article keys * 2,016 panes * 4 bytes ≈ 806 MB before framework overhead"
    thirty_day_memory_calculation: "100K active article keys * 8,640 panes * 4 bytes ≈ 3.46 GB before framework overhead"
    counter_width_note: "The 4 byte calculation is an illustrative lower-bound estimate, so use a wider counter type when counts can exceed 2^32"

// Illustrative fixed-capacity Space-Saving tracker for one leaderboard context.
// Production Flink state should store the sketch in managed state so checkpoints can recover it.
class SpaceSavingTopK {
  private readonly capacity = 1000;
  private entries: Map<string, number> = new Map();

  onShareEvent(articleId: string): void {
    if (this.entries.has(articleId)) {
      // Case 1: Candidate already tracked, increment counter
      this.entries.set(articleId, (this.entries.get(articleId) ?? 0) + 1);
    } else if (this.entries.size < this.capacity) {
      // Case 2: Available capacity, insert candidate with initial count of 1
      this.entries.set(articleId, 1);
    } else {
      // Case 3: Saturated capacity, evict minimum entry and increment its score
      const [minArticleId, minCount] = this.findMinimumEntry();
      this.entries.delete(minArticleId);
      this.entries.set(articleId, minCount + 1);
    }
  }

  // The sketch bounds frequency overestimation, but boundary top-K membership is approximate.
  // This reference implementation scans K entries, so each update is O(K).
  // A production min-heap can reduce minimum lookup/update work to O(log K).
  // Reset the sketch when the 5 minute pane closes, as window state is built from recent pane summaries.
  private findMinimumEntry(): [string, number] {
    let minKey = "";
    let minVal = Number.POSITIVE_INFINITY;
    for (const [key, val] of this.entries.entries()) {
      if (val < minVal) {
        minVal = val;
        minKey = key;
      }
    }
    return [minKey, minVal];
  }
}

hotspot_scenario:
  trigger: "Breaking news event generates 1M shares in 10 minutes"
  failure_mode: "All write increments hit a single Flink keyed task and Redis counter shard, causing worker backpressure"

remediation_strategies:
  gateway_microbatch:
    description: "API servers coalesce serving-counter updates in local memory for 1 second"
    batch_flush:
      article_id: "art-123"
      count: 342
    durability_note: "Raw share events continue to Kafka, ensuring gateway memory batching is never the only durable copy"
    result: "Reduces Redis counter update QPS for bursty hot articles"

  sub_key_sharding:
    description: "Partition counter updates using hash(event_id) % 8 across 8 deterministic sub-keys"
    stage_two_merge: "Downstream Flink aggregation sums partial counts across all 8 sub-keys"
    ordering_note: "Global per-article order is relaxed in sharded mode because share counting is commutative, while identical event retries stay on the same shard"
    result: "Distributes a single viral article's counter load across multiple cluster nodes"

topic_configuration:
  name: "share-events"
  partitions: 64
  partition_key: "article_id"
  ordering: "Preserves per-article order in the baseline unsalted path"
  hot_key_mode: "For sharded viral articles, use article_id + shard_id and rely on commutative counting rather than global ordering"
  retention_days: 7
  replication_factor: 3
  min_insync_replicas: 2

producer:
  source: "Share API after deterministic URL normalization, where unresolved shorteners may use a provisional article_id"
  idempotency: "Derive one event_id per authenticated user plus idempotency key, so retries return the same event_id within the retention window"
  idempotency_retention: "Keep the request idempotency record for at least the maximum expected client retry window"
  acks: "all"
  enable_idempotence: true
  event_schema:
    event_id: "string"
    article_id: "string"
    canonical_url: "string"
    user_id: "string"
    platform: "string"
    region: "string"
    category: "string"
    timestamp: "int64"

consumer_groups:
  window_counter:
    engine: "Apache Flink"
    responsibility: "Maintains 5 minute pane counts supporting 1h, 24h, 7d, and 30d windows per article, region, and category"
  topk_aggregator:
    engine: "Apache Flink"
    responsibility: "Maintains bounded candidate summaries for active 5 minute panes, merges candidates from the relevant panes for each leaderboard context, and flushes ranked results to Redis every 60 seconds"
  persistence_writer:
    engine: "ClickHouse Ingest Worker"
    responsibility: "Uses durable idempotency state keyed by event_id, retries safely, and executes micro-batch INSERT every 10 seconds into ClickHouse for exact historical reconciliation and auditing"

operational_paths:
  read_path: "GET /api/v1/top-articles -> Queries Redis ZREVRANGE topk:24h:US:tech 0 99 WITHSCORES"
  dead_letter_queue:
    topic: "share-events-dlq"
    reasons: "Poison events after bounded retries, schema failures, or unrecoverable processing errors"
  consumer_lag_alert:
    threshold: "60 seconds"