Machine Coding Problem

A/B Testing Library

maco60macoAllinfrastructurehashinguser-bucketing
Commonly Asked By:NetflixMetaOptimizelyGoogle

Functional Scope (In-Scope)

  • Deterministic User Bucketing: Map a userId to a consistent bucket (0–99) using cryptographic hashing.
  • Weight-Based Variant Allocation: Route buckets to experimental variants (e.g., A/B/C) based on defined traffic weights.
  • Forced User Overrides: Allow force-assigning specific test accounts to specific variants for internal QA.
  • Deduplicated Exposure Logging: Record when a user is exposed to a variant, ensuring each exposure is logged only once per user.

Explicit Boundaries (Out-of-Scope)

  • No Real-time Database Persistence Sync: Bypasses live, low-latency database sync protocols for tracking.
  • No Statistical Significance Testing: Excludes running offline t-tests or p-value statistical pipelines in real time.

Clean reference designs demonstrating deterministic bucketing in Java and Python:

// ─── JAVA BLUEPRINT ──────────────────────────────────────────────────────────
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.*;
import java.util.concurrent.*;

enum ExperimentStatus { DRAFT, RUNNING, STOPPED }

// Represents an experimental variant configuration
class Variant {
    private final String name;
    private final int weight; // Weight out of 100
    private final Map<String, Object> variables;

    public Variant(String name, int weight, Map<String, Object> variables) {
        this.name = name;
        this.weight = weight;
        this.variables = Collections.unmodifiableMap(variables);
    }

    public String getName() { return name; }
    public int getWeight() { return weight; }
    public Map<String, Object> getVariables() { return variables; }

    @Override
    public String toString() { return name + "(" + weight + "%)"; }
}

// Pluggable Hashing Strategy (Open-Closed Principle)
interface HashStrategy {
    int getHashValue(String key);
}

class MD5HashStrategy implements HashStrategy {
    @Override
    public int getHashValue(String key) {
        try {
            MessageDigest md = MessageDigest.getInstance("MD5");
            byte[] bytes = md.digest(key.getBytes(StandardCharsets.UTF_8));
            // Read last 4 bytes as integer
            int val = ((bytes[12] & 0xFF) << 24) |
                      ((bytes[13] & 0xFF) << 16) |
                      ((bytes[14] & 0xFF) << 8)  |
                      (bytes[15] & 0xFF);
            return Math.abs(val);
        } catch (NoSuchAlgorithmException e) {
            return Math.abs(key.hashCode());
        }
    }
}

// Experiment definition holding metadata and configs
class Experiment {
    private final String id;
    private final String name;
    private final List<Variant> variants = new CopyOnWriteArrayList<>();
    private final Map<String, String> overrides = new ConcurrentHashMap<>();
    private ExperimentStatus status = ExperimentStatus.DRAFT;

    public Experiment(String id, String name) {
        this.id = id;
        this.name = name;
    }

    public void addVariant(Variant v) {
        variants.add(v);
    }

    public void start() {
        int totalWeight = variants.stream().mapToInt(Variant::getWeight).sum();
        if (totalWeight != 100) {
            throw new IllegalStateException("Experiment variant weights must sum to exactly 100%. Current sum: " + totalWeight);
        }
        this.status = ExperimentStatus.RUNNING;
    }

    public void stop() {
        this.status = ExperimentStatus.STOPPED;
    }

    public void addOverride(String userId, String variantName) {
        overrides.put(userId, variantName);
    }

    public String getId() { return id; }
    public String getName() { return name; }
    public List<Variant> getVariants() { return variants; }
    public Map<String, String> getOverrides() { return overrides; }
    public ExperimentStatus getStatus() { return status; }
}

// Exposure logging to track metrics with in-memory deduplication
class ExposureLogger {
    private final Set<String> exposureLogs = ConcurrentHashMap.newKeySet();

    public void logExposure(String userId, String experimentId, String variantName) {
        String logKey = userId + ":" + experimentId;
        if (exposureLogs.add(logKey)) {
            // First time exposure
            System.out.println("[EXPOSURE LOG] User: " + userId + " -> Experiment: " + 
                experimentId + " -> Variant: " + variantName);
        }
    }
}

// High performance bucketing routing engine
class ABTestingEngine {
    private final HashStrategy hashStrategy;
    private final ExposureLogger logger;
    private final Map<String, Experiment> experiments = new ConcurrentHashMap<>();

    public ABTestingEngine(HashStrategy hashStrategy, ExposureLogger logger) {
        this.hashStrategy = hashStrategy;
        this.logger = logger;
    }

    public void registerExperiment(Experiment exp) {
        experiments.put(exp.getId(), exp);
    }

    public Variant evaluate(String userId, String experimentId) {
        Experiment exp = experiments.get(experimentId);
        if (exp == null || exp.getStatus() != ExperimentStatus.RUNNING) {
            return null; // No active experiment
        }

        // 1. Force override (QA bypass)
        String overrideVariantName = exp.getOverrides().get(userId);
        if (overrideVariantName != null) {
            for (Variant v : exp.getVariants()) {
                if (v.getName().equals(overrideVariantName)) {
                    logger.logExposure(userId, experimentId, v.getName());
                    return v;
                }
            }
        }

        // 2. Hash-based bucketing
        String key = userId + ":" + experimentId;
        int bucket = hashStrategy.getHashValue(key) % 100;

        int cumulative = 0;
        for (Variant variant : exp.getVariants()) {
            cumulative += variant.getWeight();
            if (bucket < cumulative) {
                logger.logExposure(userId, experimentId, variant.getName());
                return variant;
            }
        }
        return null;
    }
}

public class Main {
    public static void main(String[] args) {
        System.out.println("=== INITIALIZING A/B TESTING ENGINE ===");
        ExposureLogger logger = new ExposureLogger();
        ABTestingEngine engine = new ABTestingEngine(new MD5HashStrategy(), logger);

        // 1. Set up an experiment
        Experiment checkoutExp = new Experiment("exp_checkout_button", "New Checkout Button Colors");
        Map<String, Object> controlVars = new HashMap<>(); controlVars.put("color", "#CCCCCC");
        Map<String, Object> redVars = new HashMap<>(); redVars.put("color", "#FF0000");
        Map<String, Object> blueVars = new HashMap<>(); blueVars.put("color", "#0000FF");

        checkoutExp.addVariant(new Variant("Control", 50, controlVars));
        checkoutExp.addVariant(new Variant("Treatment_Red", 25, redVars));
        checkoutExp.addVariant(new Variant("Treatment_Blue", 25, blueVars));

        // Set QA forced overrides
        checkoutExp.addOverride("qa_user_1", "Treatment_Blue");
        checkoutExp.addOverride("qa_user_2", "Treatment_Red");

        engine.registerExperiment(checkoutExp);
        checkoutExp.start();

        System.out.println("\n=== 2. QA OVERRIDE TESTS ===");
        Variant qa1 = engine.evaluate("qa_user_1", "exp_checkout_button");
        System.out.println("QA User 1 Variant: " + qa1.getName());

        System.out.println("\n=== 3. DEDUPLICATION PROOF (Lookup again) ===");
        // The second lookup must NOT print [EXPOSURE LOG]
        Variant qa1SecondTime = engine.evaluate("qa_user_1", "exp_checkout_button");
        System.out.println("QA User 1 Second Lookup: " + qa1SecondTime.getName());

        System.out.println("\n=== 4. MASS POPULATION WEIGHT BALANCING SIMULATION ===");
        Map<String, Integer> variantDistribution = new ConcurrentHashMap<>();
        int simulationCount = 10000;

        for (int i = 0; i < simulationCount; i++) {
            String userId = "user_" + i;
            Variant assigned = engine.evaluate(userId, "exp_checkout_button");
            if (assigned != null) {
                variantDistribution.merge(assigned.getName(), 1, Integer::sum);
            }
        }

        System.out.println("\nSimulation completed for " + simulationCount + " users.");
        for (Map.Entry<String, Integer> entry : variantDistribution.entrySet()) {
            double percent = ((double) entry.getValue() / simulationCount) * 100;
            System.out.printf("  - Variant: %-15s | Assigned: %-4d | Ratio: %.2f%%\n", 
                entry.getKey(), entry.getValue(), percent);
        }
    }
}