Toxicity Labels

const detector = new ToxicityDetector({
  labels: ['insult', 'threat', 'toxicity'],
  threshold: 0.8,
});

const detector = new ToxicityDetector({
  labels: ['insult', 'identity_attack', 'sexual_explicit', 'threat'],
  threshold: 0.85,
});

const detector = new ToxicityDetector({
  labels: ToxicityDetector.ALL_LABELS, // All 7 categories
  threshold: 0.8,
});

const detector = new ToxicityDetector({
  labels: ['threat', 'severe_toxicity'],
  threshold: 0.7, // Lower threshold for safety
});

import { ToxicityDetector } from 'glin-profanity/ml';

async function handleByCategory(text: string) {
  const detector = new ToxicityDetector({ threshold: 0.8 });
  const result = await detector.analyze(text);

  if (!result.isToxic) {
    return { action: 'allow' };
  }

  // Handle by severity
  if (result.matchedCategories.includes('threat') ||
      result.matchedCategories.includes('severe_toxicity')) {
    return {
      action: 'block_and_report',
      escalate: true,
      reason: 'Potential threat detected',
    };
  }

  if (result.matchedCategories.includes('identity_attack')) {
    return {
      action: 'block',
      warn: true,
      reason: 'Hate speech policy violation',
    };
  }

  if (result.matchedCategories.includes('insult')) {
    return {
      action: 'warn',
      reason: 'Please keep discussions respectful',
    };
  }

  // Default handling
  return {
    action: 'flag_for_review',
    categories: result.matchedCategories,
  };
}