☰

GPT Comment Detector

Identify AI-generated comments on Hacker News

  1. // ==UserScript==
  2. // @name         GPT Comment Detector
  3. // @namespace    https://example.com/
  4. // @version      0.1
  5. // @description  Identify AI-generated comments on Hacker News
  6. // @author       chryzsh
  7. // @match        https://news.ycombinator.com/*
  8. // @grant        none
  9. // @connect      huggingface.co
  10. // @grant         GM.xmlHttpRequest
  11. // @grant		unsafeWindow
  12. // @sandbox		JavaScript
  13. // @license MIT
  14. // ==/UserScript==
  15.  
  16. // Set a threshold for the minimum probability that a comment is AI-generated
  17. const AI_THRESHOLD = 0.99;
  18. const PROBABLE_THRESHOLD = 0.7;
  19. const TOKEN_THRESHOLD = 200; //4 characters are about 50 tokens, which is when the detector model because reliable
  20.  
  21. // put all comments into a NodeList
  22. const comments = document.querySelectorAll('.comment');
  23.  
  24. // regex to remove whitespaces
  25. const whitespaceRegex = /\s+/g;
  26.  
  27. //regex to remove "reply" at the end of comments
  28. const replyRegex = /\breply\b\s*/g;
  29.  
  30. // kick off the program
  31. main();
  32.  
  33. function main() {
  34.   // loop over each comment
  35.   comments.forEach((comment) => {
  36.     // add an onclick for each comment
  37.     comment.addEventListener('click', function () {
  38.       // trim commments for whitespaces and test length for each comment
  39.       testLength(comment);
  40.     });
  41.   });
  42. };
  43.  
  44. // Trim and test the length of comments
  45. function testLength(comment) {
  46.   var text = comment.textContent;
  47.  
  48.   // trim whitespaces and shit from the end first - dont do this on the comment object itself, just a variable
  49.   text = comment.textContent.trim();
  50.   text = comment.textContent.replace(whitespaceRegex, " ");
  51.  
  52.   // remove "reply" from the end
  53.   text = comment.textContent.replace(replyRegex, "");
  54.  
  55.   // test the length first so we don't run detection on too short comments
  56.   if (text.length < TOKEN_THRESHOLD) {
  57.     comment.style.border = '1px solid gray';
  58.     comment.innerHTML += `<div style="color: gray; font-weight: bold;">Insufficient data to assess if AI generated</div>`;
  59.   }
  60.   else {
  61.     // run AI detection and add comment based on the result
  62.     detectAI(comment.textContent, function (result) {
  63.       addComment(result, comment);
  64.     });
  65.  
  66.   }
  67. }
  68.  
  69. // Add comment
  70. function addComment(result, comment) {
  71.   var json = JSON.parse(result);
  72.   var fakeProbability = json.fake_probability;
  73.   var realProbability = json.real_probability;
  74.   var formattedNumberAI = (fakeProbability * 100).toFixed(3);
  75.   var formattedNumberHuman = (realProbability * 100).toFixed(3);
  76.  
  77.   // most comments will not be AI (hopefully) so start with testing that
  78.   if (fakeProbability < PROBABLE_THRESHOLD) { //assume not AI if lower than 0.7 on the AI-meter
  79.     comment.style.border = '1px solid green';
  80.     comment.innerHTML += `<div style="color: green; font-weight: bold;">Definitely human - human probability ${formattedNumberHuman}</div>`;
  81.   }
  82.   // If the probability is above the threshold, label the comment as AI-generated
  83.   else if (fakeProbability > PROBABLE_THRESHOLD) { // possibly AI
  84.     if (fakeProbability > AI_THRESHOLD) { //definitely AI
  85.  
  86.       comment.style.border = '1px solid red';
  87.       comment.innerHTML += `<div style="color: red; font-weight: bold;">Definitely AI - AI probability ${formattedNumberAI}</div>`;
  88.     }
  89.     else {
  90.       comment.style.border = '1px solid yellow';
  91.       comment.innerHTML += `<div style="color: yellow; font-weight: bold;">Possibly AI - AI probability ${formattedNumberAI}</div>`;
  92.     }
  93.   }
  94. };
  95.  
  96. // Run AI detection
  97. function detectAI(input, callback) {
  98.   var detectorUrlWithInput = "https://huggingface.co/openai-detector?" + input
  99.   GM.xmlHttpRequest({
  100.     method: "GET",
  101.     url: detectorUrlWithInput,
  102.     headers: {
  103.       "User-Agent": "Mozilla/5.0",    // If not specified, navigator.userAgent will be used.
  104.       "Content-Type": "application/x-www-form-urlencoded"
  105.     },
  106.  
  107.     onload: function (response) {
  108.       var responseParsed = [response.responseText].join("\n");
  109.       callback(responseParsed);
  110.     }
  111.   });
  112. }