ToxicityDetector

import { ToxicityDetector } from 'glin-profanity/ml';

async function moderateContent(text: string) {
  const detector = new ToxicityDetector({ threshold: 0.85 });

  // Preload model for faster response
  await detector.loadModel();

  const result = await detector.analyze(text);

  if (result.isToxic) {
    console.log('Toxic content detected!');
    console.log('Categories:', result.matchedCategories.join(', '));
    console.log('Score:', result.overallScore.toFixed(2));
    return { allowed: false, reason: result.matchedCategories };
  }

  return { allowed: true };
}

// Usage
const result = await moderateContent('you are an idiot');
// { allowed: false, reason: ['insult', 'toxicity'] }

import { ToxicityDetector } from 'glin-profanity/ml';

class ChatModerator {
  private detector: ToxicityDetector;
  private initialized = false;

  constructor() {
    this.detector = new ToxicityDetector({
      threshold: 0.8,
      labels: ['insult', 'threat', 'toxicity'],
    });
  }

  async init() {
    if (!this.initialized) {
      await this.detector.loadModel();
      this.initialized = true;
    }
  }

  async checkMessage(message: string) {
    await this.init();

    const result = await this.detector.analyze(message);

    return {
      allowed: !result.isToxic,
      score: result.overallScore,
      categories: result.matchedCategories,
      processingTime: result.processingTimeMs,
    };
  }
}

// Singleton instance
const moderator = new ChatModerator();

// Usage in message handler
app.post('/chat', async (req, res) => {
  const check = await moderator.checkMessage(req.body.message);

  if (!check.allowed) {
    return res.status(400).json({
      error: 'Message rejected',
      reason: check.categories
    });
  }

  // Process message...
});

import { ToxicityDetector } from 'glin-profanity/ml';

async function moderateComments(comments: string[]) {
  const detector = new ToxicityDetector({ threshold: 0.85 });
  await detector.loadModel();

  // Batch process for efficiency
  const results = await detector.analyzeBatch(comments);

  return comments.map((comment, i) => ({
    text: comment,
    isToxic: results[i].isToxic,
    score: results[i].overallScore,
    categories: results[i].matchedCategories,
  }));
}

// Usage
const comments = [
  'Great article, thanks for sharing!',
  'You are such a moron',
  'I disagree with your point about taxes',
  'Kill yourself loser',
];

const moderated = await moderateComments(comments);
moderated.forEach((result) => {
  console.log(`"${result.text.substring(0, 30)}..." - ${result.isToxic ? 'BLOCKED' : 'OK'}`);
});