Deploy machine learning models and forensic analysis to detect synthetic media
Deepfakes use generative AI to create synthetic media that is increasingly difficult to distinguish from authentic content. Detection requires sophisticated analysis across multiple modalities and scales.
Convolutional Neural Networks learn to identify subtle artifacts in synthetic images:
// Example: CNN Deepfake Detector with TensorFlow.js
import * as tf from '@tensorflow/tfjs-node';
class CNNDeepfakeDetector {
constructor() {
this.model = this.buildModel();
}
buildModel() {
const model = tf.sequential();
// Feature extraction layers
model.add(tf.layers.conv2d({
inputShape: [224, 224, 3],
filters: 32,
kernelSize: 3,
activation: 'relu'
}));
model.add(tf.layers.maxPooling2d({ poolSize: 2 }));
model.add(tf.layers.dropout({ rate: 0.25 }));
model.add(tf.layers.conv2d({ filters: 64, kernelSize: 3, activation: 'relu' }));
model.add(tf.layers.maxPooling2d({ poolSize: 2 }));
model.add(tf.layers.dropout({ rate: 0.25 }));
model.add(tf.layers.conv2d({ filters: 128, kernelSize: 3, activation: 'relu' }));
model.add(tf.layers.maxPooling2d({ poolSize: 2 }));
model.add(tf.layers.dropout({ rate: 0.25 }));
// Dense layers
model.add(tf.layers.flatten());
model.add(tf.layers.dense({ units: 256, activation: 'relu' }));
model.add(tf.layers.dropout({ rate: 0.5 }));
// Output layer (binary classification)
model.add(tf.layers.dense({
units: 1,
activation: 'sigmoid'
}));
model.compile({
optimizer: 'adam',
loss: 'binaryCrossentropy',
metrics: ['accuracy']
});
return model;
}
async detect(imageTensor) {
// Preprocess image
const normalized = imageTensor.div(255.0);
const resized = tf.image.resizeBilinear(normalized, [224, 224]);
const batched = resized.expandDims(0);
// Get prediction
const prediction = this.model.predict(batched);
const score = await prediction.data();
const result = {
is_deepfake: score[0] > 0.5,
confidence: Math.abs(score[0] - 0.5) * 2, // 0 to 1
score: score[0],
method: 'CNN Binary Classifier'
};
// Cleanup tensors
batched.dispose();
resized.dispose();
normalized.dispose();
prediction.dispose();
return result;
}
async analyzeAttentionMap(imageTensor) {
// Generate attention map showing suspicious regions
const features = this.extractFeatures(imageTensor);
const attention = this.computeGradCAM(features);
return attention;
}
extractFeatures(imageTensor) {
// Extract intermediate layer activations
const featureModel = tf.model({
inputs: this.model.input,
outputs: this.model.getLayer('conv2d_2').output
});
return featureModel.predict(imageTensor);
}
}
// Usage
const detector = new CNNDeepfakeDetector();
const result = await detector.detect(imageTensor);
console.log('Deepfake detected:', result.is_deepfake);
console.log('Confidence:', (result.confidence * 100).toFixed(1) + '%');
Vision Transformers excel at capturing long-range dependencies and global inconsistencies:
// Example: Vision Transformer for Deepfake Detection
class ViTDeepfakeDetector {
constructor(patchSize = 16, numLayers = 12) {
this.patchSize = patchSize;
this.numLayers = numLayers;
this.model = this.buildTransformer();
}
buildTransformer() {
// Simplified ViT architecture
const input = tf.input({ shape: [224, 224, 3] });
// Patch embedding
const patches = this.patchify(input);
const embedded = tf.layers.dense({
units: 768,
name: 'patch_embedding'
}).apply(patches);
// Positional encoding
const positioned = this.addPositionalEncoding(embedded);
// Transformer encoder blocks
let x = positioned;
for (let i = 0; i < this.numLayers; i++) {
x = this.transformerBlock(x, i);
}
// Classification head
const pooled = tf.layers.globalAveragePooling1d().apply(x);
const output = tf.layers.dense({
units: 1,
activation: 'sigmoid',
name: 'classification'
}).apply(pooled);
return tf.model({ inputs: input, outputs: output });
}
transformerBlock(x, blockId) {
// Multi-head self-attention
const attention = tf.layers.multiHeadAttention({
numHeads: 12,
keyDim: 64,
name: `attention_${blockId}`
}).apply(x, x);
// Add & Norm
const normed1 = tf.layers.layerNormalization({
name: `norm1_${blockId}`
}).apply(tf.layers.add().apply([x, attention]));
// Feed-forward network
const ff1 = tf.layers.dense({
units: 3072,
activation: 'gelu',
name: `ff1_${blockId}`
}).apply(normed1);
const ff2 = tf.layers.dense({
units: 768,
name: `ff2_${blockId}`
}).apply(ff1);
// Add & Norm
const output = tf.layers.layerNormalization({
name: `norm2_${blockId}`
}).apply(tf.layers.add().apply([normed1, ff2]));
return output;
}
async detectWithExplanation(imageTensor) {
const prediction = await this.model.predict(imageTensor);
const score = await prediction.data();
// Extract attention maps for explanation
const attentionMaps = await this.extractAttentionMaps(imageTensor);
return {
is_deepfake: score[0] > 0.5,
confidence: Math.abs(score[0] - 0.5) * 2,
attention_maps: attentionMaps,
suspicious_patches: this.identifySuspiciousPatches(attentionMaps)
};
}
}
Traditional forensic techniques detect inconsistencies that betray manipulation:
// Example: Error Level Analysis
async function errorLevelAnalysis(imagePath) {
const original = await Jimp.read(imagePath);
// Save at specific JPEG quality
const tempPath = '/tmp/ela_temp.jpg';
await original.quality(95).writeAsync(tempPath);
// Reload compressed version
const compressed = await Jimp.read(tempPath);
// Compute pixel-wise difference
const ela = new Jimp(original.bitmap.width, original.bitmap.height);
original.scan(0, 0, original.bitmap.width, original.bitmap.height, (x, y, idx) => {
const origR = original.bitmap.data[idx];
const compR = compressed.bitmap.data[idx];
const diff = Math.abs(origR - compR);
const amplified = Math.min(255, diff * 10); // Amplify differences
ela.bitmap.data[idx] = amplified;
ela.bitmap.data[idx + 1] = amplified;
ela.bitmap.data[idx + 2] = amplified;
ela.bitmap.data[idx + 3] = 255;
});
return {
ela_image: ela,
anomaly_score: computeAnomalyScore(ela),
suspicious_regions: detectSuspiciousRegions(ela)
};
}
function computeAnomalyScore(elaImage) {
let totalDiff = 0;
let pixelCount = 0;
elaImage.scan(0, 0, elaImage.bitmap.width, elaImage.bitmap.height, (x, y, idx) => {
totalDiff += elaImage.bitmap.data[idx];
pixelCount++;
});
return totalDiff / pixelCount / 255;
}
Different cameras produce different noise patterns. Inconsistent noise suggests manipulation:
// Example: Camera Noise Pattern Analysis
class NoiseAnalyzer {
async analyzeNoise(imagePath) {
const image = await Jimp.read(imagePath);
// Extract noise using wavelet denoising
const noise = await this.extractNoise(image);
// Compute noise characteristics
const stats = this.computeNoiseStatistics(noise);
// Check for inconsistencies
const inconsistencies = this.detectInconsistencies(noise, stats);
return {
noise_level: stats.stdDev,
noise_pattern: stats.pattern,
inconsistent_regions: inconsistencies,
tampering_likelihood: inconsistencies.length > 0 ? 'HIGH' : 'LOW'
};
}
async extractNoise(image) {
// Apply median filter to estimate image without noise
const denoised = this.medianFilter(image, 3);
// Subtract to get noise
const noise = new Jimp(image.bitmap.width, image.bitmap.height);
image.scan(0, 0, image.bitmap.width, image.bitmap.height, (x, y, idx) => {
const diff = image.bitmap.data[idx] - denoised.bitmap.data[idx];
noise.bitmap.data[idx] = 128 + diff; // Center around 128
noise.bitmap.data[idx + 3] = 255;
});
return noise;
}
detectInconsistencies(noise, globalStats) {
const blockSize = 32;
const inconsistencies = [];
// Divide into blocks and check local statistics
for (let y = 0; y < noise.bitmap.height; y += blockSize) {
for (let x = 0; x < noise.bitmap.width; x += blockSize) {
const blockStats = this.computeBlockStatistics(noise, x, y, blockSize);
// Check if block statistics deviate significantly
if (Math.abs(blockStats.stdDev - globalStats.stdDev) / globalStats.stdDev > 0.3) {
inconsistencies.push({
x, y,
width: blockSize,
height: blockSize,
deviation: Math.abs(blockStats.stdDev - globalStats.stdDev)
});
}
}
}
return inconsistencies;
}
}
Analyzing multiple modalities improves detection accuracy:
// Example: Multi-Modal Deepfake Detector
class MultiModalDetector {
constructor() {
this.visualDetector = new CNNDeepfakeDetector();
this.audioDetector = new AudioDeepfakeDetector();
this.syncDetector = new LipSyncDetector();
}
async analyzeVideo(videoPath) {
// Extract frames and audio
const frames = await extractFrames(videoPath);
const audio = await extractAudio(videoPath);
// Visual analysis
const visualResults = [];
for (const frame of frames.slice(0, 10)) { // Sample frames
const result = await this.visualDetector.detect(frame);
visualResults.push(result.score);
}
const visualScore = average(visualResults);
// Audio analysis
const audioResult = await this.audioDetector.analyze(audio);
const audioScore = audioResult.synthetic_probability;
// Lip sync analysis
const syncResult = await this.syncDetector.analyze(videoPath);
const syncScore = syncResult.mismatch_score;
// Combine scores
const finalScore = (visualScore * 0.4 +
audioScore * 0.3 +
syncScore * 0.3);
return {
is_deepfake: finalScore > 0.5,
confidence: Math.abs(finalScore - 0.5) * 2,
breakdown: {
visual: visualScore,
audio: audioScore,
lip_sync: syncScore
},
suspicious_frames: this.identifySuspiciousFrames(visualResults)
};
}
}
class LipSyncDetector {
async analyze(videoPath) {
// Extract facial landmarks and audio features
const landmarks = await extractLandmarks(videoPath);
const audioFeatures = await extractMFCC(videoPath);
// Check synchronization
let totalMismatch = 0;
const syncWindow = 3; // frames
for (let i = 0; i < landmarks.length; i++) {
const mouthMovement = this.computeMouthMovement(
landmarks.slice(i, i + syncWindow)
);
const audioEnergy = this.computeAudioEnergy(
audioFeatures.slice(i, i + syncWindow)
);
// Correlation should be high for natural speech
const correlation = this.correlate(mouthMovement, audioEnergy);
if (correlation < 0.6) {
totalMismatch += (0.6 - correlation);
}
}
return {
mismatch_score: totalMismatch / landmarks.length,
synchronized: totalMismatch < landmarks.length * 0.1
};
}
}
Different GAN architectures leave unique fingerprints in generated images:
// Example: GAN Attribution
class GANAttributor {
constructor() {
this.knownModels = [
'StyleGAN2',
'StyleGAN3',
'DALL-E',
'Midjourney',
'Stable Diffusion'
];
this.classifier = this.loadAttributionModel();
}
async attributeModel(imageTensor) {
// Extract GAN-specific artifacts
const features = await this.extractGANFeatures(imageTensor);
// Classify to specific model
const predictions = await this.classifier.predict(features);
const probabilities = await predictions.data();
const results = this.knownModels.map((model, idx) => ({
model: model,
probability: probabilities[idx]
})).sort((a, b) => b.probability - a.probability);
return {
most_likely: results[0].model,
confidence: results[0].probability,
all_probabilities: results
};
}
async extractGANFeatures(imageTensor) {
// Frequency domain analysis
const fft = await this.computeFFT(imageTensor);
// Specific artifact patterns
const artifacts = {
checkerboard: this.detectCheckerboard(fft),
spectral_peaks: this.detectSpectralPeaks(fft),
color_correlation: this.analyzeColorCorrelation(imageTensor)
};
return tf.tensor([
artifacts.checkerboard,
artifacts.spectral_peaks,
artifacts.color_correlation
]);
}
}
Deepfakes often fail to replicate subtle biological signals:
Combining multiple detectors improves robustness:
// Example: Ensemble Detector
class EnsembleDeepfakeDetector {
constructor() {
this.detectors = [
new CNNDeepfakeDetector(),
new ViTDeepfakeDetector(),
new ForensicAnalyzer(),
new GANAttributor(),
new BiologicalSignalDetector()
];
}
async detect(media) {
const results = [];
// Run all detectors
for (const detector of this.detectors) {
try {
const result = await detector.detect(media);
results.push({
detector: detector.constructor.name,
score: result.is_deepfake ? result.confidence : (1 - result.confidence),
details: result
});
} catch (error) {
console.error(`Detector ${detector.constructor.name} failed:`, error);
}
}
// Weighted voting
const weights = [0.3, 0.25, 0.2, 0.15, 0.1]; // Adjust based on detector reliability
let weightedScore = 0;
let totalWeight = 0;
results.forEach((result, idx) => {
weightedScore += result.score * weights[idx];
totalWeight += weights[idx];
});
const finalScore = weightedScore / totalWeight;
return {
is_deepfake: finalScore > 0.5,
confidence: Math.abs(finalScore - 0.5) * 2,
ensemble_score: finalScore,
individual_results: results,
consensus: results.filter(r => r.score > 0.5).length / results.length
};
}
}
εΌηδΊΊι (νμ΅μΈκ°) Β· Benefit All Humanity