Jump to content

User:Crestfalling/AdvancedDeduplicate.js

From Wikipedia, the free encyclopedia
Note: After saving, you have to bypass your browser's cache to see the changes. Google Chrome, Firefox, Microsoft Edge and Safari: Hold down the ⇧ Shift key and click the Reload toolbar button. For details and instructions about other browsers, see Wikipedia:Bypass your cache.
(function () {
  'use strict';

  const DEBUG = true;
  const debug = (...args) => {
    if (DEBUG) console.log('[DeduplicateReferences]', ...args);
  };

  const SCRIPT_LINK = '[[User:Crestfalling/AdvancedDeduplicate.js|AdvancedDeduplicate]]';

  const ALLOWED_NAMESPACES = new Set([0, 118]);

  const DOMAIN_BLACKLIST = new Set([
    'doi.org', 'dx.doi.org', 'jstor.org', 'amazon.com',
    'books.google.com', 'web.archive.org', 'archive.org',
    'worldcat.org', 'patents.google.com',
  ]);

  const NAME_BLACKLIST = new Set([
    'cite_journal', 'cite_book', 'cite_web', 'cite_news', 'cite_magazine',
    'cite_newspaper', 'cite_thesis', 'cite_conference', 'cite_encyclopedia',
    'cite_album_notes', 'cite_comic', 'cite_court', 'cite_act', 'cite_episode',
    'cite_mailing_list', 'cite_map', 'cite_newsgroup', 'cite_patent',
    'cite_press_release', 'cite_report', 'cite_video_game', 'citation',
  ]);

  function normalizeDiacritics(str) {
    return str.normalize('NFD').replace(/[\u0300-\u036f]/g, '');
  }

  const PROTECTED_TAGS = ['nowiki', 'pre', 'syntaxhighlight', 'source', 'math'];

  function findProtectedRanges(text) {
    const ranges = [];
    const commentRe = /<!--[\s\S]*?-->/g;
    let m;

    while ((m = commentRe.exec(text))) {
      ranges.push([m.index, m.index + m[0].length]);
    }

    for (const tag of PROTECTED_TAGS) {
      const re = new RegExp(`<${tag}\\b[^>]*>[\\s\\S]*?<\\/${tag}>`, 'gi');
      while ((m = re.exec(text))) {
        ranges.push([m.index, m.index + m[0].length]);
      }
    }

    ranges.sort((a, b) => a[0] - b[0]);
    return ranges;
  }

  function inProtectedRange(pos, ranges) {
    for (const [s, e] of ranges) {
      if (s > pos) break;
      if (pos >= s && pos < e) return true;
    }
    return false;
  }

  function parseRefs(text) {
    const tokens = [];
    const protectedRanges = findProtectedRanges(text);
    let i = 0;

    while (i < text.length) {
      const s = text.indexOf('<ref', i);
      if (s === -1) break;

      if (inProtectedRange(s, protectedRanges)) {
        i = s + 4;
        continue;
      }

      const c = text[s + 4];
      if (c !== '>' && c !== '/' && !/\s/.test(c || '')) {
        i = s + 4;
        continue;
      }

      const gt = text.indexOf('>', s);
      if (gt === -1) {
        i = s + 1;
        continue;
      }

      const openTag = text.slice(s, gt + 1);

      if (openTag.endsWith('/>')) {
        tokens.push({ start: s, end: gt + 1, raw: openTag, selfClose: true });
        i = gt + 1;
      } else {
        const closeStart = text.indexOf('</ref>', gt + 1);
        if (closeStart === -1) {
          i = gt + 1;
          continue;
        }
        const end = closeStart + 6;
        tokens.push({ start: s, end, raw: text.slice(s, end), selfClose: false });
        i = end;
      }
    }

    return tokens;
  }

  function attrValue(raw, attrName) {
    const openMatch = raw.match(/^<ref[^>]*>/i) || raw.match(/^<ref[^>]*\/>/i);
    const scope = openMatch ? openMatch[0] : raw;
    const re = new RegExp(
      `\\b${attrName}\\s*=\\s*(?:"([^"]*)"|'([^']*)'|“([^”]*)”|‘([^’]*)’|([^\\s/>]*))`,
      'i'
    );
    const m = scope.match(re);
    if (!m) return null;
    const val = (m[1] || m[2] || m[3] || m[4] || m[5] || '').trim();
    return val === '' ? null : val;
  }

  const getName = raw => attrValue(raw, 'name');
  const getGroup = raw => attrValue(raw, 'group');

  function isEffectivelyNamed(raw) {
    if (getName(raw) !== null) return true;
    const openMatch = raw.match(/^<ref[^>]*>/i) || raw.match(/^<ref[^>]*\/>/i);
    const scope = openMatch ? openMatch[0] : raw;
    return /\bname\s*=/i.test(scope);
  }

  const getContent = raw => {
    const m = raw.match(/^<ref[^>]*>([\s\S]*)<\/ref>$/i);
    return m ? m[1] : '';
  };

  const NAME_ATTR_RE = /\s*\bname\s*=\s*(?:"[^"]*"|'[^']*'|[^\s/>]*)/gi;

  function normaliseParam(p) {
    return p.replace(/\s+/g, ' ').trim().replace(/\s*=\s*/, '=');
  }

  function extractTemplate(content) {
    const start = content.search(/\{\{\s*(?:cite|citation)/i);
    if (start === -1) return null;

    const parts = [];
    let current = '';
    let depth = 0;
    let end = -1;

    for (let i = start; i < content.length; i++) {
      const ch = content[i];
      const next = content[i + 1] || '';

      if ((ch === '{' && next === '{') || (ch === '[' && next === '[')) {
        depth++;
        current += ch + next;
        i++;
      } else if ((ch === '}' && next === '}') || (ch === ']' && next === ']')) {
        depth--;
        current += ch + next;
        i++;
        if (depth === 0) {
          end = i + 1;
          break;
        }
      } else if (ch === '|' && depth === 1) {
        parts.push(current);
        current = '';
      } else {
        current += ch;
      }
    }

    if (end === -1) return null;
    parts.push(current);
    return { start, end, parts };
  }

  function sortTemplateParams(content) {
    const t = extractTemplate(content);
    if (!t) return content;
    const { start, end, parts: params } = t;

    if (params.length === 1) {
      const name = params[0]
        .replace(/^\{\{/, '').replace(/\}\}$/, '')
        .replace(/\s+/g, ' ').trim();
      return content.slice(0, start) + '{{' + name + '}}' + content.slice(end);
    }

    const head = params[0].replace(/^\{\{/, '').replace(/\s+/g, ' ').trim();
    const tail = params[params.length - 1].replace(/\}\}$/, '');
    const sorted = params.slice(1, -1).concat([tail])
      .map(normaliseParam)
      .filter(Boolean)
      .sort();

    return content.slice(0, start) +
      '{{' + head + '|' + sorted.join('|') + '}}' +
      content.slice(end);
  }

  function normalise(raw) {
    const stripped = raw.replace(NAME_ATTR_RE, '');

    const m = stripped.match(/^<ref([^>]*)>([\s\S]*)<\/ref>$/i);
    if (!m) {
      return normalizeDiacritics(stripped.replace(/\s+/g, ' ').trim());
    }

    const attrs = m[1].replace(/\s+/g, ' ').trim();
    let content = m[2];

    if (/\{\{\s*(?:cite|citation)/i.test(content)) {
      content = sortTemplateParams(content);
    }
    content = content.replace(/\s+/g, ' ').trim();

    return normalizeDiacritics(`<ref ${attrs}>${content}</ref>`);
  }

  const URL_RE = /https?:\/\/[^\s|<>\]}"']+/i;
  const BARE_URL_RE = /^\[?\s*(https?:\/\/[^\s\]<>|]+)\s*\]?$/i;

  function normaliseUrl(u) {
    let s = u.trim()
      .replace(/#.*$/, '')
      .replace(/^[a-z][a-z0-9+.-]*:\/\//i, '')
      .replace(/^\/\//, '')
      .replace(/^www\./i, '');
    const slash = s.indexOf('/');
    const host = (slash === -1 ? s : s.slice(0, slash)).toLowerCase();
    const rest = slash === -1 ? '' : s.slice(slash);
    return (host + rest).replace(/\/+$/, '');
  }

  const KEY_ALIASES = new Map([
    ['journal', 'work'], ['magazine', 'work'], ['newspaper', 'work'],
    ['periodical', 'work'], ['website', 'work'],
    ['last', 'last1'], ['surname', 'last1'], ['surname1', 'last1'],
    ['author-last', 'last1'], ['author-last1', 'last1'], ['author1-last', 'last1'],
    ['first', 'first1'], ['given', 'first1'], ['given1', 'first1'],
    ['author-first', 'first1'], ['author-first1', 'first1'], ['author1-first', 'first1'],
    ['archiveurl', 'archive-url'], ['archivedate', 'archive-date'],
    ['accessdate', 'access-date'], ['lang', 'language'],
  ]);
  const canonicalKey = k => KEY_ALIASES.get(k) || k;

	function normalizeComparisonValue(value) {
		return normalizeDiacritics(
			value
			.replace(/\[\[([^\]|]+)\|([^\]]+)\]\]/g, '$2')
			.replace(/\[\[([^\]]+)\]\]/g, '$1')
			.replace(/\s+/g, ' ')
			.trim()
		);
	}
	
  function templateInfo(content) {
    const t = extractTemplate(content);
    if (!t) return null;

    const whole = content.slice(0, t.start).trim() === '' &&
      content.slice(t.end).trim() === '';

    const parts = t.parts.slice();
    const head = parts[0]
      .replace(/^\{\{/, '').replace(/\}\}$/, '')
      .replace(/\s+/g, ' ').trim().toLowerCase();

    const params = new Map();
    let exactOnly = false;

    if (parts.length > 1) {
      parts[parts.length - 1] = parts[parts.length - 1].replace(/\}\}$/, '');
      for (const p of parts.slice(1)) {
        if (p.trim() === '') continue;
        const eq = p.indexOf('=');
        if (eq === -1) {
          exactOnly = true;
          continue;
        }
        const k = p.slice(0, eq).replace(/\s+/g, ' ').trim().toLowerCase();
        const v = normalizeDiacritics(p.slice(eq + 1).replace(/\s+/g, ' ').trim());
        if (!/^[\w\- ]+$/.test(k)) {
          exactOnly = true;
          continue;
        }
        if (v) params.set(k, v);
      }
    }

	const canon = new Map();
	params.forEach((v, k) => {
		canon.set(canonicalKey(k), normalizeComparisonValue(v));
	});

    return { head, params, canon, whole, exactOnly };
  }

  const MONTH_PREFIXES = ['jan', 'feb', 'mar', 'apr', 'may', 'jun', 'jul', 'aug', 'sep', 'oct', 'nov', 'dec'];

  function parseLooseDate(str) {
    const v = String(str || '').replace(/<!--[\s\S]*?-->/g, '').replace(/\s+/g, ' ').trim();
    if (!v) return null;

    const month = name => {
      const i = MONTH_PREFIXES.indexOf(name.toLowerCase().slice(0, 3));
      return i === -1 ? null : i + 1;
    };

    let m, y, mo, d;
    if ((m = v.match(/^(\d{4})[-/](\d{1,2})[-/](\d{1,2})$/))) {
      y = +m[1]; mo = +m[2]; d = +m[3];
    } else if ((m = v.match(/^(\d{1,2}) ([A-Za-z]+)\.?,? (\d{4})$/))) {
      y = +m[3]; mo = month(m[2]); d = +m[1];
    } else if ((m = v.match(/^([A-Za-z]+)\.? (\d{1,2}),? (\d{4})$/))) {
      y = +m[3]; mo = month(m[1]); d = +m[2];
    } else if ((m = v.match(/^([A-Za-z]+)\.? (\d{4})$/))) {
      y = +m[2]; mo = month(m[1]); d = 1;
    } else {
      return null;
    }

    if (!mo || mo > 12 || d < 1 || d > 31) return null;
    return Date.UTC(y, mo - 1, d);
  }

  function statusOf(d) {
    return d.tpl ? (d.tpl.params.get('url-status') || '').toLowerCase() : '';
  }

  function statusCompatible(a, b) {
    const sa = statusOf(a), sb = statusOf(b);
    if (sa === sb) return true;
    if ((sa === 'live' && sb === '') || (sb === 'live' && sa === '')) {
      return a.accessTs !== null && b.accessTs !== null && a.accessTs !== b.accessTs;
    }
    return sa === '';
  }

  function removeUrlStatusLive(content) {
    const t = extractTemplate(content);
    if (!t) return content;
    const sub = content.slice(t.start, t.end)
      .replace(/\s*\|\s*url-status\s*=\s*live(?=\s*(?:\||\}\}))/i, '');
    return content.slice(0, t.start) + sub + content.slice(t.end);
  }

  function addTemplateParam(content, text) {
    const t = extractTemplate(content);
    if (!t) return content;
    let i = t.end - 2;
    while (i > t.start && /\s/.test(content[i - 1])) i--;
    const spaced = /\s\|/.test(content.slice(t.start, t.end));
    return content.slice(0, i) + (spaced ? ' |' : '|') + text + content.slice(i);
  }

  const ACCESS_DATE_RE = /(\|\s*)(access-?date)(\s*=\s*)([^|}]*?)(\s*(?=\||\}\}))/i;

  function rawAccessDate(d) {
    const content = getContent(d.tok.raw);
    const t = extractTemplate(content);
    if (!t) return null;
    const m = content.slice(t.start, t.end).match(ACCESS_DATE_RE);
    if (!m || !m[4].trim() || /[{}\[\]]/.test(m[4])) return null;
    return { name: m[2], value: m[4].trim() };
  }

  function setAccessDate(content, name, value) {
    const t = extractTemplate(content);
    if (!t) return content;
    const sub = content.slice(t.start, t.end);
    if (ACCESS_DATE_RE.test(sub)) {
      return content.slice(0, t.start) +
        sub.replace(ACCESS_DATE_RE, (all, pipe, nm, eq, old, tail) => pipe + nm + eq + value + tail) +
        content.slice(t.end);
    }
    return addTemplateParam(content, name + '=' + value);
  }

  function applyRecency(content, keeper, cluster) {
    if (keeper.kind !== 'template') return content;
    const dated = cluster.filter(m => m.kind === 'template' && m.accessTs !== null);
    if (dated.length === 0) return content;

    const newest = Math.max.apply(null, dated.map(m => m.accessTs));
    const latest = dated.filter(m => m.accessTs === newest)
      .sort((x, y) => x.tok.start - y.tok.start);

    const wanted = latest.map(statusOf).filter((v, i, arr) => arr.indexOf(v) === i);
    if (wanted.length === 1) {
      const want = wanted[0], have = statusOf(keeper);
      if (want === 'live' && have === '') content = addTemplateParam(content, 'url-status=live');
      else if (want === '' && have === 'live') content = removeUrlStatusLive(content);
    }

    const keeperHasUnparseableDate = keeper.accessTs === null &&
      !!(keeper.tpl.params.get('access-date') || keeper.tpl.params.get('accessdate'));
    if (!keeperHasUnparseableDate && (keeper.accessTs === null || keeper.accessTs < newest)) {
      const raw = rawAccessDate(latest[0]);
      if (raw) content = setAccessDate(content, raw.name, raw.value);
    }
    return content;
  }

  function describeRef(tok) {
    const content = getContent(tok.raw);
    const tpl = templateInfo(content);
    const group = getGroup(tok.raw) || '';

    let kind = 'opaque';
    let url = null;
    let score = 0;

    if (tpl && tpl.whole && !tpl.exactOnly) {
      kind = 'template';
      score = tpl.params.size;
      url = tpl.params.get('url') || null;
    } else {
      const bare = content.trim().match(BARE_URL_RE);
      if (bare) {
        kind = 'bare';
        url = bare[1];
      } else {
        const m = content.match(URL_RE);
        if (m) url = m[0];
      }
    }

    const norm = normalise(tok.raw);

    return {
      tok,
      kind,
      tpl: kind === 'template' ? tpl : null,
      ids: kind === 'template' ? idKeys(tpl.params) : [],
      group,
      urlKey: url ? normaliseUrl(url) : null,
      accessTs: kind === 'template'
        ? parseLooseDate(tpl.params.get('access-date') || tpl.params.get('accessdate'))
        : null,
      norm,
      score,
      len: norm.length,
    };
  }

  const ID_KEYS = ['pmid', 'pmc', 'doi', 'isbn', 'jstor', 'arxiv', 'bibcode'];

  function idKeys(params) {
    const out = [];
    ID_KEYS.forEach(k => {
      let v = params.get(k);
      if (!v) return;
      v = v.toLowerCase().replace(/\s+/g, '');
      if (k === 'pmc') v = v.replace(/^pmc/, '');
      if (k === 'isbn') v = v.replace(/-/g, '');
      if (k === 'doi') v = v.replace(/^https?:\/\/(?:dx\.)?doi\.org\//, '');
      if (v) out.push(k + ':' + v);
    });
    return out;
  }

  const PAGE_KEYS = ['page', 'pages', 'p', 'pp'];
  const OTHER_LOCATOR_KEYS = ['at', 'quote', 'section', 'chapter'];
  const LOCATOR_KEYS = PAGE_KEYS.concat(OTHER_LOCATOR_KEYS);

  function normalisePageItem(item) {
    const text = item.replace(/^pp?\.|^pp?(?=\d)/, '');
    const parts = text.split(/[-–—−]/).filter(Boolean);
    if (parts.length < 2) return text;

    const start = parts[0];
    const end = parts[parts.length - 1];
    const m1 = /^([a-z]*)(\d+)$/.exec(start);
    const m2 = /^([a-z]*)(\d+)$/.exec(end);

    let fullEnd = end;
    if (m1 && m2) {
      let digits = m2[2];
      if (!m2[1] && digits.length < m1[2].length) {
        digits = m1[2].slice(0, m1[2].length - digits.length) + digits;
      }
      fullEnd = (m2[1] || m1[1]) + digits;
    }
    return fullEnd === start ? start : start + '-' + fullEnd;
  }

  function normalisePages(value) {
    return value.toLowerCase().replace(/\s+/g, '')
      .split(/[,;]/).filter(Boolean)
      .map(normalisePageItem).join(',');
  }

  function locatorSignature(params) {
    let page = '';
    PAGE_KEYS.some(k => {
      if (params.has(k)) {
        page = params.get(k);
        return true;
      }
      return false;
    });
    const pages = normalisePages(page).split(',').filter(Boolean).sort().join(',');
    const rest = OTHER_LOCATOR_KEYS
      .map(k => (params.get(k) || '').toLowerCase().replace(/\s+/g, ''))
      .join('|');
    return pages + '|' + rest;
  }

  const NO_LOCATOR = '||||';

  const VOLATILE_KEYS = ['access-date', 'accessdate', 'df'];

  function isSubset(a, b, lenientStatus) {
    if (a.norm === b.norm) return true;

    if (a.kind === 'bare') {
      return a.urlKey !== null && a.urlKey === b.urlKey &&
        b.kind === 'template' && locatorSignature(b.tpl.params) === NO_LOCATOR;
    }
    if (a.kind !== 'template' || b.kind !== 'template') return false;
    if (locatorSignature(a.tpl.params) !== locatorSignature(b.tpl.params)) return false;

    if (a.ids.some(id => b.ids.indexOf(id) !== -1)) return true;

    if (a.urlKey === null || a.urlKey !== b.urlKey) return false;
    for (const [k, v] of a.tpl.canon) {
      if (k === 'url' || LOCATOR_KEYS.indexOf(k) !== -1 || VOLATILE_KEYS.indexOf(k) !== -1) continue;
      if (lenientStatus && k === 'url-status') continue;
      if (b.tpl.canon.get(k) !== v) return false;
    }
    return !lenientStatus || statusCompatible(a, b);
  }

  function slug(s) {
    return normalizeDiacritics(s)
      .replace(/[^A-Za-z0-9]+/g, '_')
      .replace(/^_+|_+$/g, '')
      .toLowerCase()
      .slice(0, 40);
  }

  function uniqueName(base, used) {
    let name = base;
    for (let n = 2; used.has(name); n++) name = `${base}_${n}`;
    used.add(name);
    return name;
  }

  function uniqueAuthorName(base, used) {
    if (!used.has(base)) {
      used.add(base);
      return base;
    }
    for (let c = 97; c <= 122; c++) {
      const cand = base + String.fromCharCode(c);
      if (!used.has(cand)) {
        used.add(cand);
        return cand;
      }
    }
    return uniqueName(base, used);
  }

  function citeParamsRaw(content) {
    const out = new Map();
    const t = extractTemplate(content);
    if (!t) return out;
    const parts = t.parts.slice();
    parts[parts.length - 1] = parts[parts.length - 1].replace(/\}\}$/, '');
    for (const p of parts.slice(1)) {
      const eq = p.indexOf('=');
      if (eq === -1) continue;
      const k = p.slice(0, eq).replace(/\s+/g, ' ').trim().toLowerCase();
      const v = p.slice(eq + 1).replace(/\s+/g, ' ').trim();
      if (k && v) out.set(k, v);
    }
    return out;
  }

  const ORG_WORD_RE = /\b(?:news|press|staff|team|inc|ltd|llc|corp|company|association|university|institute|department|bureau|agency|committee|council|group|office|services?|society|foundation|ministry|government|times|post|journal|magazine|board|commission|authority|organi[sz]ation|museum|library|project|network)\b/i;

  const SURNAME_KEYS = ['last', 'last1', 'surname', 'surname1', 'author-last', 'author-last1', 'author1-last'];
  const AUTHOR_KEYS = ['author', 'author1', 'authors'];

  function keepNameChar(ch) {
    if (/[A-Za-z' -]/.test(ch) || ch === '\u2019') return true;
    const c = ch.charCodeAt(0);
    if (c < 192 || c === 0xD7 || c === 0xF7) return false;
    if (c >= 0x2000 && c <= 0x2BFF) return false;
    if (c >= 0x3000 && c <= 0x303F) return false;
    return true;
  }

  function cleanSurname(s) {
    const v = s
      .replace(/\[\[(?:[^\]|]*\|)?([^\]]*)\]\]/g, '$1')
      .replace(/<[^>]*>/g, '')
      .replace(/\s+/g, ' ')
      .trim();
    if (!v || /[\d{}\[\]|=<>\"]/.test(v)) return null;
    const out = v.split('').filter(keepNameChar).join('').replace(/\s+/g, ' ').trim();
    return out ? out.slice(0, 40) : null;
  }

  function surnameFromAuthor(value, vancouver) {
    const seg = value.split(/\s*(?:;|&|\band\b)\s*/i)[0].trim();
    if (!seg || ORG_WORD_RE.test(seg)) return null;
    if (vancouver) return cleanSurname(seg.split(/\s*,\s*/)[0].split(' ')[0]);
    if (seg.includes(',')) {
      const words = seg.split(',')[0].trim().split(' ');
      if (words.length > 1 && /^[A-Z]{1,3}\.?$/.test(words[words.length - 1])) words.pop();
      return cleanSurname(words.join(' '));
    }
    const words = seg.split(' ');
    if (words.length === 1) return cleanSurname(words[0]);
    if (words.length > 3 || words.some(w => /^[A-Z]{2,}$/.test(w))) return null;
    return cleanSurname(words[words.length - 1]);
  }

  function nameFromAuthorYear(content) {
    const p = citeParamsRaw(content);
    if (!p.size) return null;

    let surname = null;
    for (const k of SURNAME_KEYS) {
      if (p.has(k)) {
        surname = cleanSurname(p.get(k));
        break;
      }
    }
    if (!surname) {
      for (const k of AUTHOR_KEYS) {
        if (p.has(k)) {
          surname = surnameFromAuthor(p.get(k), false);
          break;
        }
      }
    }
    if (!surname && p.has('vauthors')) surname = surnameFromAuthor(p.get('vauthors'), true);
    if (!surname) return null;

    for (const k of ['year', 'date', 'publication-date']) {
      const m = p.has(k) && p.get(k).match(/(?:^|\D)(1\d{3}|20\d{2})(?!\d)/);
      if (m) return `${surname} ${m[1]}`;
    }
    return null;
  }

  function nameFromCiteTitle(content) {
    const m = content.match(/\|\s*title\s*=\s*([^|{}\n]+)/i);
    if (!m) return null;

    const words = normalizeDiacritics(m[1].replace(/<[^>]*>/g, '').trim())
      .match(/\b[A-Za-z0-9]+\b/g);
    return (words && words.length) ? slug(words.slice(0, 3).join('_')) : null;
  }

  function nameFromUrl(content) {
    const m = content.match(/https?:\/\/([^\s/<>\"]+)/i);
    if (!m) return null;

    let host = m[1].replace(/^www\./, '');

    if (/^(web\.)?archive\.org/.test(host)) {
      const arc = content.match(
        /archive\.org\/web\/\d+\/https?:\/\/(?:www\.)?([^\s/<>\"]+)/i
      );
      if (arc) host = arc[1];
    }

    const domain = host.split('/')[0];
    return DOMAIN_BLACKLIST.has(domain) ? null : slug(domain);
  }

  function nameFromText(content) {
    const plain = normalizeDiacritics(content
      .replace(/<[^>]*>/g, '')
      .replace(/\[\[([^|\]]*)[^\]]*\]\]/g, '$1')
      .trim());
    const words = plain.match(/\b[A-Za-z0-9]+\b/g);
    return (words && words.length) ? slug(words.slice(0, 3).join('_')) : null;
  }

  function generateName(raw, used) {
    const content = getContent(raw);
    const hasCite = /\{\{\s*(?:cite|citation)/i.test(content);

    if (hasCite) {
      const authorYear = nameFromAuthorYear(content);
      if (authorYear) return uniqueAuthorName(authorYear, used);
    }

    const candidates = [];
    if (hasCite) candidates.push(nameFromCiteTitle(content));
    candidates.push(nameFromUrl(content));
    if (!hasCite) candidates.push(nameFromCiteTitle(content));
    candidates.push(nameFromText(content));

    const base = candidates.find(c => c && !NAME_BLACKLIST.has(c)) || 'ref';
    return uniqueName(base, used);
  }

  function plainName(raw) {
    const open = (raw.match(/^<ref[^>]*>/i) || [raw])[0];
    const m = open.match(/\bname\s*=\s*(?:"([^\"]+)"|'([^']+)'|([^\s"'“”‘’\/>][^\s"'“”‘’\/>]*))/i);
    if (!m) return null;
    const v = (m[1] || m[2] || m[3] || '').trim();
    return v && v.indexOf('"') === -1 ? v : null;
  }

  function deduplicate(text) {
    const tokens = parseRefs(text);
    debug('Found', tokens.length, 'reference tags');

    const usedNames = new Set();
    for (const t of tokens) {
      const n = getName(t.raw);
      if (n) {
        usedNames.add(n);
        const s = slug(n);
        if (s) usedNames.add(s);
      }
    }

    const replacements = [];
    let count = 0;
    let partial = 0;

    const allUnnamed = [];
    const namedTargets = [];
    for (const tok of tokens) {
      if (tok.selfClose) continue;
      if (isEffectivelyNamed(tok.raw)) {
        const name = plainName(tok.raw);
        if (name && !/\bfollow\s*=/i.test(tok.raw.match(/^<ref[^>]*>/i)[0])) {
          namedTargets.push({ name, d: describeRef(tok) });
        }
        continue;
      }
      allUnnamed.push(describeRef(tok));
    }

    const refs = [];
    for (const r of allUnnamed) {
      const hits = namedTargets.filter(t => t.d.group === r.group && isSubset(r, t.d));
      if (hits.length === 0) {
        refs.push(r);
        continue;
      }
      hits.sort((x, y) =>
        ((x.d.norm === r.norm ? 0 : 1) - (y.d.norm === r.norm ? 0 : 1)) ||
        (y.d.score - x.d.score) || (x.d.tok.start - y.d.tok.start));
      const t = hits[0];
      const gAttr = r.group ? ` group="${r.group}"` : '';
      debug('Folding unnamed ref into existing named ref', t.name);
      replacements.push({
        start: r.tok.start,
        end: r.tok.end,
        text: `<ref name="${t.name}"${gAttr} />`,
      });
      count++;
      if (r.norm !== t.d.norm) partial++;
    }

    const parent = refs.map((_, i) => i);
    const root = i => {
      while (parent[i] !== i) {
        parent[i] = parent[parent[i]];
        i = parent[i];
      }
      return i;
    };
    const owner = new Map();
    refs.forEach((r, i) => {
      const keys = r.ids.map(id => 'i:' + id);
      if (r.urlKey !== null) keys.push('u:' + r.urlKey);
      if (keys.length === 0) keys.push('n:' + r.norm);
      keys.forEach(k => {
        const key = r.group + '\u0000' + k;
        if (owner.has(key)) parent[root(i)] = root(owner.get(key));
        else owner.set(key, i);
      });
    });

    const buckets = new Map();
    refs.forEach((r, i) => {
      const k = root(i);
      if (!buckets.has(k)) buckets.set(k, []);
      buckets.get(k).push(r);
    });

    debug('Reference buckets:', buckets.size);

    for (const entries of buckets.values()) {
      if (entries.length < 2) continue;

      const ordered = entries.slice().sort((a, b) =>
        (b.score - a.score) || (b.len - a.len) || (a.tok.start - b.tok.start));

      const clusters = [];
      for (const e of ordered) {
        const home = clusters.find(c => isSubset(e, c[0], true));
        if (home) home.push(e);
        else clusters.push([e]);
      }

      for (const cluster of clusters) {
        if (cluster.length < 2) continue;

        const keeper = cluster[0];
        const members = cluster.slice().sort((a, b) => a.tok.start - b.tok.start);
        const first = members[0];

        const name = generateName(keeper.tok.raw, usedNames);
        const group = getGroup(keeper.tok.raw);
        const gAttr = group ? ` group="${group}"` : '';
        const content = applyRecency(getContent(keeper.tok.raw), keeper, cluster);

        debug('Cluster of', cluster.length, '->', name);

        replacements.push({
          start: first.tok.start,
          end: first.tok.end,
          text: `<ref name="${name}"${gAttr}>${content}</ref>`,
        });

        for (const m of members) {
          if (m === first) continue;
          replacements.push({
            start: m.tok.start,
            end: m.tok.end,
            text: `<ref name="${name}"${gAttr} />`,
          });
          count++;
        }

        for (const m of members) {
          if (m !== keeper && m.norm !== keeper.norm) partial++;
        }
      }
    }

    if (count === 0) return { text, count: 0, partial: 0 };

    replacements.sort((a, b) => b.start - a.start);
    let result = text;
    for (const r of replacements) {
      result = result.slice(0, r.start) + r.text + result.slice(r.end);
    }

    return { text: result, count, partial };
  }

  function findOrphanReferences(text) {
    const tokens = parseRefs(text);
    const defined = new Set();
    const used = new Set();

    for (const t of tokens) {
      const name = getName(t.raw);
      if (name === null) continue;
      const key = name + '\u0000' + (getGroup(t.raw) || '');
      if (t.selfClose) used.add(key);
      else defined.add(key);
    }

    const orphans = [];
    for (const key of used) {
      if (!defined.has(key)) {
        const [name, group] = key.split('\u0000');
        orphans.push(group ? `${name} (group="${group}")` : name);
      }
    }
    return orphans;
  }

  function isCurrentRevision() {
    const p = new URLSearchParams(window.location.search);
    if (p.get('oldid')) {
      debug('Editing old revision (oldid=' + p.get('oldid') + '), skipping');
      return false;
    }
    if (p.get('diff')) {
      debug('In diff view, skipping');
      return false;
    }

    const warning = document.querySelector('.mw-revision-warning, .mw-editnotice-base');
    if (warning && warning.textContent.toLowerCase().includes('old revision')) {
      debug('Old revision warning detected, skipping');
      return false;
    }
    return true;
  }

  function runOnPage(manual = false) {
    debug('Starting deduplication process...');
    if (!isCurrentRevision()) return;

    const ta = document.getElementById('wpTextbox1');
    const sum = document.getElementById('wpSummary');
    if (!ta || !sum) {
      debug('Edit textarea or summary input not found');
      if (manual) {
        alert('DeduplicateReferences: Could not find the edit textarea or edit summary.');
      }
      return;
    }

    // Use MediaWiki's textSelection API so edits are applied to CodeMirror too.
    const $ta = $(ta);
    const original = $ta.textSelection('getContents');
    const { text, count, partial } = deduplicate(original);
    debug('Deduplication complete. Count:', count, 'partial:', partial);

    if (count === 0) {
      if (manual) alert('No duplicate references found to deduplicate.');
      return;
    }

    const beforeOrphans = new Set(findOrphanReferences(original));
    const newOrphans = findOrphanReferences(text).filter(o => !beforeOrphans.has(o));

    if (newOrphans.length > 0) {
      console.error('[DeduplicateReferences] Aborting, would create orphan reference(s):', newOrphans);
      alert(
        'DeduplicateReferences aborted: this change would leave the following ' +
        'named reference(s) invoked but never defined: ' + newOrphans.join(', ') + '. ' +
        'No changes were made. Please report this at the script talk page with the article title.'
      );
      return;
    }

    // Update the visible editor, whether CodeMirror or the plain textarea is active.
    $ta.textSelection('setContents', text);

    const plural = count !== 1 ? 's' : '';
    const partialNote = partial > 0
      ? ` (${partial} merged into a more complete citation of the same URL)`
      : '';
    const msg = `[[WP:NAMEDREFS|Deduplicated]] ${count} reference${plural}${partialNote} using ${SCRIPT_LINK}`;
    sum.value = sum.value ? `${sum.value} • ${msg}` : msg;

    const minor = document.querySelector('#wpMinoredit, input[name="wpMinoredit"]');
    if (minor) minor.checked = true;

    const diffBtn = document.querySelector('#wpDiff, input[name="wpDiff"]');
    const shownDiff = !!diffBtn;
    if (diffBtn) diffBtn.click();

    alert(
      `Deduplicated ${count} reference${plural}` +
      (partial > 0 ? `, ${partial} of them partial matches merged into a fuller citation.` : '.') +
      (shownDiff
        ? ' Review the diff now loading below before publishing.'
        : ' Please review the textarea before publishing.')
    );
  }

  function addPortletLink() {
    if (document.getElementById('t-dedupe')) return;

    mw.loader.using(['mediawiki.util', 'jquery.textSelection']).then(() => {
      if (document.getElementById('t-dedupe')) return;

      const item = mw.util.addPortletLink(
        'p-tb',
        '#',
        'Deduplicate references',
        't-dedupe',
        'Merge duplicate references in the edit box'
      );
      if (!item) {
        debug('Tools portlet not found, link not added');
        return;
      }

      const link = item.querySelector('a');
      if (!link) {
        debug('Could not find portlet anchor');
        return;
      }

      link.addEventListener('click', e => {
        e.preventDefault();
        debug('Portlet clicked');
        if (!isCurrentRevision()) {
          alert('Deduplication only works when editing the current version of an article.');
          return;
        }
        runOnPage(true);
      });
      debug('Added deduplication portlet link');
    }).catch(error => {
      console.error('[DeduplicateReferences] Could not load required MediaWiki modules:', error);
    });
  }

  function whenLoaded(fn) {
    if (document.readyState === 'complete') {
      fn();
    } else {
      window.addEventListener('load', fn);
    }
  }

  function init() {
    if (typeof mw === 'undefined') {
      console.log('DeduplicateReferences: MediaWiki not found, aborting.');
      return;
    }

    const wgAction = mw.config.get('wgAction');
    const wgNs = mw.config.get('wgNamespaceNumber');

    const shouldRun = (wgAction === 'edit' || wgAction === 'submit') &&
      ALLOWED_NAMESPACES.has(wgNs);

    if (shouldRun) {
      debug('Article/draft edit page detected, setting up deduplication...');
      whenLoaded(addPortletLink);
    } else {
      debug('Not on article/draft edit page - action:', wgAction, 'namespace:', wgNs);
    }
  }

  init();
})();