User:Crestfalling/AdvancedDeduplicate.js
Appearance
Code that you insert on this page could contain malicious content capable of compromising your account. If you import a script from another page with "importScript", "mw.loader.load", "iusc", or "lusc", take note that this causes you to dynamically load a remote script, which could be changed by others. Editors are responsible for all edits and actions they perform, including by scripts. User scripts are not centrally supported and may malfunction or become inoperable due to software changes. A guide to help you find broken scripts is available. If you are unsure whether code you are adding to this page is safe, you can ask at the appropriate village pump.
This code will be executed when previewing this page.
This code will be executed when previewing this page.
This user script seems to have a documentation page at User:Crestfalling/AdvancedDeduplicate.
(function () {
'use strict';
const DEBUG = true;
const debug = (...args) => {
if (DEBUG) console.log('[DeduplicateReferences]', ...args);
};
const SCRIPT_LINK = '[[User:Crestfalling/AdvancedDeduplicate.js|AdvancedDeduplicate]]';
const ALLOWED_NAMESPACES = new Set([0, 118]);
const DOMAIN_BLACKLIST = new Set([
'doi.org', 'dx.doi.org', 'jstor.org', 'amazon.com',
'books.google.com', 'web.archive.org', 'archive.org',
'worldcat.org', 'patents.google.com',
]);
const NAME_BLACKLIST = new Set([
'cite_journal', 'cite_book', 'cite_web', 'cite_news', 'cite_magazine',
'cite_newspaper', 'cite_thesis', 'cite_conference', 'cite_encyclopedia',
'cite_album_notes', 'cite_comic', 'cite_court', 'cite_act', 'cite_episode',
'cite_mailing_list', 'cite_map', 'cite_newsgroup', 'cite_patent',
'cite_press_release', 'cite_report', 'cite_video_game', 'citation',
]);
function normalizeDiacritics(str) {
return str.normalize('NFD').replace(/[\u0300-\u036f]/g, '');
}
const PROTECTED_TAGS = ['nowiki', 'pre', 'syntaxhighlight', 'source', 'math'];
function findProtectedRanges(text) {
const ranges = [];
const commentRe = /<!--[\s\S]*?-->/g;
let m;
while ((m = commentRe.exec(text))) {
ranges.push([m.index, m.index + m[0].length]);
}
for (const tag of PROTECTED_TAGS) {
const re = new RegExp(`<${tag}\\b[^>]*>[\\s\\S]*?<\\/${tag}>`, 'gi');
while ((m = re.exec(text))) {
ranges.push([m.index, m.index + m[0].length]);
}
}
ranges.sort((a, b) => a[0] - b[0]);
return ranges;
}
function inProtectedRange(pos, ranges) {
for (const [s, e] of ranges) {
if (s > pos) break;
if (pos >= s && pos < e) return true;
}
return false;
}
function parseRefs(text) {
const tokens = [];
const protectedRanges = findProtectedRanges(text);
let i = 0;
while (i < text.length) {
const s = text.indexOf('<ref', i);
if (s === -1) break;
if (inProtectedRange(s, protectedRanges)) {
i = s + 4;
continue;
}
const c = text[s + 4];
if (c !== '>' && c !== '/' && !/\s/.test(c || '')) {
i = s + 4;
continue;
}
const gt = text.indexOf('>', s);
if (gt === -1) {
i = s + 1;
continue;
}
const openTag = text.slice(s, gt + 1);
if (openTag.endsWith('/>')) {
tokens.push({ start: s, end: gt + 1, raw: openTag, selfClose: true });
i = gt + 1;
} else {
const closeStart = text.indexOf('</ref>', gt + 1);
if (closeStart === -1) {
i = gt + 1;
continue;
}
const end = closeStart + 6;
tokens.push({ start: s, end, raw: text.slice(s, end), selfClose: false });
i = end;
}
}
return tokens;
}
function attrValue(raw, attrName) {
const openMatch = raw.match(/^<ref[^>]*>/i) || raw.match(/^<ref[^>]*\/>/i);
const scope = openMatch ? openMatch[0] : raw;
const re = new RegExp(
`\\b${attrName}\\s*=\\s*(?:"([^"]*)"|'([^']*)'|“([^”]*)”|‘([^’]*)’|([^\\s/>]*))`,
'i'
);
const m = scope.match(re);
if (!m) return null;
const val = (m[1] || m[2] || m[3] || m[4] || m[5] || '').trim();
return val === '' ? null : val;
}
const getName = raw => attrValue(raw, 'name');
const getGroup = raw => attrValue(raw, 'group');
function isEffectivelyNamed(raw) {
if (getName(raw) !== null) return true;
const openMatch = raw.match(/^<ref[^>]*>/i) || raw.match(/^<ref[^>]*\/>/i);
const scope = openMatch ? openMatch[0] : raw;
return /\bname\s*=/i.test(scope);
}
const getContent = raw => {
const m = raw.match(/^<ref[^>]*>([\s\S]*)<\/ref>$/i);
return m ? m[1] : '';
};
const NAME_ATTR_RE = /\s*\bname\s*=\s*(?:"[^"]*"|'[^']*'|[^\s/>]*)/gi;
function normaliseParam(p) {
return p.replace(/\s+/g, ' ').trim().replace(/\s*=\s*/, '=');
}
function extractTemplate(content) {
const start = content.search(/\{\{\s*(?:cite|citation)/i);
if (start === -1) return null;
const parts = [];
let current = '';
let depth = 0;
let end = -1;
for (let i = start; i < content.length; i++) {
const ch = content[i];
const next = content[i + 1] || '';
if ((ch === '{' && next === '{') || (ch === '[' && next === '[')) {
depth++;
current += ch + next;
i++;
} else if ((ch === '}' && next === '}') || (ch === ']' && next === ']')) {
depth--;
current += ch + next;
i++;
if (depth === 0) {
end = i + 1;
break;
}
} else if (ch === '|' && depth === 1) {
parts.push(current);
current = '';
} else {
current += ch;
}
}
if (end === -1) return null;
parts.push(current);
return { start, end, parts };
}
function sortTemplateParams(content) {
const t = extractTemplate(content);
if (!t) return content;
const { start, end, parts: params } = t;
if (params.length === 1) {
const name = params[0]
.replace(/^\{\{/, '').replace(/\}\}$/, '')
.replace(/\s+/g, ' ').trim();
return content.slice(0, start) + '{{' + name + '}}' + content.slice(end);
}
const head = params[0].replace(/^\{\{/, '').replace(/\s+/g, ' ').trim();
const tail = params[params.length - 1].replace(/\}\}$/, '');
const sorted = params.slice(1, -1).concat([tail])
.map(normaliseParam)
.filter(Boolean)
.sort();
return content.slice(0, start) +
'{{' + head + '|' + sorted.join('|') + '}}' +
content.slice(end);
}
function normalise(raw) {
const stripped = raw.replace(NAME_ATTR_RE, '');
const m = stripped.match(/^<ref([^>]*)>([\s\S]*)<\/ref>$/i);
if (!m) {
return normalizeDiacritics(stripped.replace(/\s+/g, ' ').trim());
}
const attrs = m[1].replace(/\s+/g, ' ').trim();
let content = m[2];
if (/\{\{\s*(?:cite|citation)/i.test(content)) {
content = sortTemplateParams(content);
}
content = content.replace(/\s+/g, ' ').trim();
return normalizeDiacritics(`<ref ${attrs}>${content}</ref>`);
}
const URL_RE = /https?:\/\/[^\s|<>\]}"']+/i;
const BARE_URL_RE = /^\[?\s*(https?:\/\/[^\s\]<>|]+)\s*\]?$/i;
function normaliseUrl(u) {
let s = u.trim()
.replace(/#.*$/, '')
.replace(/^[a-z][a-z0-9+.-]*:\/\//i, '')
.replace(/^\/\//, '')
.replace(/^www\./i, '');
const slash = s.indexOf('/');
const host = (slash === -1 ? s : s.slice(0, slash)).toLowerCase();
const rest = slash === -1 ? '' : s.slice(slash);
return (host + rest).replace(/\/+$/, '');
}
const KEY_ALIASES = new Map([
['journal', 'work'], ['magazine', 'work'], ['newspaper', 'work'],
['periodical', 'work'], ['website', 'work'],
['last', 'last1'], ['surname', 'last1'], ['surname1', 'last1'],
['author-last', 'last1'], ['author-last1', 'last1'], ['author1-last', 'last1'],
['first', 'first1'], ['given', 'first1'], ['given1', 'first1'],
['author-first', 'first1'], ['author-first1', 'first1'], ['author1-first', 'first1'],
['archiveurl', 'archive-url'], ['archivedate', 'archive-date'],
['accessdate', 'access-date'], ['lang', 'language'],
]);
const canonicalKey = k => KEY_ALIASES.get(k) || k;
function normalizeComparisonValue(value) {
return normalizeDiacritics(
value
.replace(/\[\[([^\]|]+)\|([^\]]+)\]\]/g, '$2')
.replace(/\[\[([^\]]+)\]\]/g, '$1')
.replace(/\s+/g, ' ')
.trim()
);
}
function templateInfo(content) {
const t = extractTemplate(content);
if (!t) return null;
const whole = content.slice(0, t.start).trim() === '' &&
content.slice(t.end).trim() === '';
const parts = t.parts.slice();
const head = parts[0]
.replace(/^\{\{/, '').replace(/\}\}$/, '')
.replace(/\s+/g, ' ').trim().toLowerCase();
const params = new Map();
let exactOnly = false;
if (parts.length > 1) {
parts[parts.length - 1] = parts[parts.length - 1].replace(/\}\}$/, '');
for (const p of parts.slice(1)) {
if (p.trim() === '') continue;
const eq = p.indexOf('=');
if (eq === -1) {
exactOnly = true;
continue;
}
const k = p.slice(0, eq).replace(/\s+/g, ' ').trim().toLowerCase();
const v = normalizeDiacritics(p.slice(eq + 1).replace(/\s+/g, ' ').trim());
if (!/^[\w\- ]+$/.test(k)) {
exactOnly = true;
continue;
}
if (v) params.set(k, v);
}
}
const canon = new Map();
params.forEach((v, k) => {
canon.set(canonicalKey(k), normalizeComparisonValue(v));
});
return { head, params, canon, whole, exactOnly };
}
const MONTH_PREFIXES = ['jan', 'feb', 'mar', 'apr', 'may', 'jun', 'jul', 'aug', 'sep', 'oct', 'nov', 'dec'];
function parseLooseDate(str) {
const v = String(str || '').replace(/<!--[\s\S]*?-->/g, '').replace(/\s+/g, ' ').trim();
if (!v) return null;
const month = name => {
const i = MONTH_PREFIXES.indexOf(name.toLowerCase().slice(0, 3));
return i === -1 ? null : i + 1;
};
let m, y, mo, d;
if ((m = v.match(/^(\d{4})[-/](\d{1,2})[-/](\d{1,2})$/))) {
y = +m[1]; mo = +m[2]; d = +m[3];
} else if ((m = v.match(/^(\d{1,2}) ([A-Za-z]+)\.?,? (\d{4})$/))) {
y = +m[3]; mo = month(m[2]); d = +m[1];
} else if ((m = v.match(/^([A-Za-z]+)\.? (\d{1,2}),? (\d{4})$/))) {
y = +m[3]; mo = month(m[1]); d = +m[2];
} else if ((m = v.match(/^([A-Za-z]+)\.? (\d{4})$/))) {
y = +m[2]; mo = month(m[1]); d = 1;
} else {
return null;
}
if (!mo || mo > 12 || d < 1 || d > 31) return null;
return Date.UTC(y, mo - 1, d);
}
function statusOf(d) {
return d.tpl ? (d.tpl.params.get('url-status') || '').toLowerCase() : '';
}
function statusCompatible(a, b) {
const sa = statusOf(a), sb = statusOf(b);
if (sa === sb) return true;
if ((sa === 'live' && sb === '') || (sb === 'live' && sa === '')) {
return a.accessTs !== null && b.accessTs !== null && a.accessTs !== b.accessTs;
}
return sa === '';
}
function removeUrlStatusLive(content) {
const t = extractTemplate(content);
if (!t) return content;
const sub = content.slice(t.start, t.end)
.replace(/\s*\|\s*url-status\s*=\s*live(?=\s*(?:\||\}\}))/i, '');
return content.slice(0, t.start) + sub + content.slice(t.end);
}
function addTemplateParam(content, text) {
const t = extractTemplate(content);
if (!t) return content;
let i = t.end - 2;
while (i > t.start && /\s/.test(content[i - 1])) i--;
const spaced = /\s\|/.test(content.slice(t.start, t.end));
return content.slice(0, i) + (spaced ? ' |' : '|') + text + content.slice(i);
}
const ACCESS_DATE_RE = /(\|\s*)(access-?date)(\s*=\s*)([^|}]*?)(\s*(?=\||\}\}))/i;
function rawAccessDate(d) {
const content = getContent(d.tok.raw);
const t = extractTemplate(content);
if (!t) return null;
const m = content.slice(t.start, t.end).match(ACCESS_DATE_RE);
if (!m || !m[4].trim() || /[{}\[\]]/.test(m[4])) return null;
return { name: m[2], value: m[4].trim() };
}
function setAccessDate(content, name, value) {
const t = extractTemplate(content);
if (!t) return content;
const sub = content.slice(t.start, t.end);
if (ACCESS_DATE_RE.test(sub)) {
return content.slice(0, t.start) +
sub.replace(ACCESS_DATE_RE, (all, pipe, nm, eq, old, tail) => pipe + nm + eq + value + tail) +
content.slice(t.end);
}
return addTemplateParam(content, name + '=' + value);
}
function applyRecency(content, keeper, cluster) {
if (keeper.kind !== 'template') return content;
const dated = cluster.filter(m => m.kind === 'template' && m.accessTs !== null);
if (dated.length === 0) return content;
const newest = Math.max.apply(null, dated.map(m => m.accessTs));
const latest = dated.filter(m => m.accessTs === newest)
.sort((x, y) => x.tok.start - y.tok.start);
const wanted = latest.map(statusOf).filter((v, i, arr) => arr.indexOf(v) === i);
if (wanted.length === 1) {
const want = wanted[0], have = statusOf(keeper);
if (want === 'live' && have === '') content = addTemplateParam(content, 'url-status=live');
else if (want === '' && have === 'live') content = removeUrlStatusLive(content);
}
const keeperHasUnparseableDate = keeper.accessTs === null &&
!!(keeper.tpl.params.get('access-date') || keeper.tpl.params.get('accessdate'));
if (!keeperHasUnparseableDate && (keeper.accessTs === null || keeper.accessTs < newest)) {
const raw = rawAccessDate(latest[0]);
if (raw) content = setAccessDate(content, raw.name, raw.value);
}
return content;
}
function describeRef(tok) {
const content = getContent(tok.raw);
const tpl = templateInfo(content);
const group = getGroup(tok.raw) || '';
let kind = 'opaque';
let url = null;
let score = 0;
if (tpl && tpl.whole && !tpl.exactOnly) {
kind = 'template';
score = tpl.params.size;
url = tpl.params.get('url') || null;
} else {
const bare = content.trim().match(BARE_URL_RE);
if (bare) {
kind = 'bare';
url = bare[1];
} else {
const m = content.match(URL_RE);
if (m) url = m[0];
}
}
const norm = normalise(tok.raw);
return {
tok,
kind,
tpl: kind === 'template' ? tpl : null,
ids: kind === 'template' ? idKeys(tpl.params) : [],
group,
urlKey: url ? normaliseUrl(url) : null,
accessTs: kind === 'template'
? parseLooseDate(tpl.params.get('access-date') || tpl.params.get('accessdate'))
: null,
norm,
score,
len: norm.length,
};
}
const ID_KEYS = ['pmid', 'pmc', 'doi', 'isbn', 'jstor', 'arxiv', 'bibcode'];
function idKeys(params) {
const out = [];
ID_KEYS.forEach(k => {
let v = params.get(k);
if (!v) return;
v = v.toLowerCase().replace(/\s+/g, '');
if (k === 'pmc') v = v.replace(/^pmc/, '');
if (k === 'isbn') v = v.replace(/-/g, '');
if (k === 'doi') v = v.replace(/^https?:\/\/(?:dx\.)?doi\.org\//, '');
if (v) out.push(k + ':' + v);
});
return out;
}
const PAGE_KEYS = ['page', 'pages', 'p', 'pp'];
const OTHER_LOCATOR_KEYS = ['at', 'quote', 'section', 'chapter'];
const LOCATOR_KEYS = PAGE_KEYS.concat(OTHER_LOCATOR_KEYS);
function normalisePageItem(item) {
const text = item.replace(/^pp?\.|^pp?(?=\d)/, '');
const parts = text.split(/[-–—−]/).filter(Boolean);
if (parts.length < 2) return text;
const start = parts[0];
const end = parts[parts.length - 1];
const m1 = /^([a-z]*)(\d+)$/.exec(start);
const m2 = /^([a-z]*)(\d+)$/.exec(end);
let fullEnd = end;
if (m1 && m2) {
let digits = m2[2];
if (!m2[1] && digits.length < m1[2].length) {
digits = m1[2].slice(0, m1[2].length - digits.length) + digits;
}
fullEnd = (m2[1] || m1[1]) + digits;
}
return fullEnd === start ? start : start + '-' + fullEnd;
}
function normalisePages(value) {
return value.toLowerCase().replace(/\s+/g, '')
.split(/[,;]/).filter(Boolean)
.map(normalisePageItem).join(',');
}
function locatorSignature(params) {
let page = '';
PAGE_KEYS.some(k => {
if (params.has(k)) {
page = params.get(k);
return true;
}
return false;
});
const pages = normalisePages(page).split(',').filter(Boolean).sort().join(',');
const rest = OTHER_LOCATOR_KEYS
.map(k => (params.get(k) || '').toLowerCase().replace(/\s+/g, ''))
.join('|');
return pages + '|' + rest;
}
const NO_LOCATOR = '||||';
const VOLATILE_KEYS = ['access-date', 'accessdate', 'df'];
function isSubset(a, b, lenientStatus) {
if (a.norm === b.norm) return true;
if (a.kind === 'bare') {
return a.urlKey !== null && a.urlKey === b.urlKey &&
b.kind === 'template' && locatorSignature(b.tpl.params) === NO_LOCATOR;
}
if (a.kind !== 'template' || b.kind !== 'template') return false;
if (locatorSignature(a.tpl.params) !== locatorSignature(b.tpl.params)) return false;
if (a.ids.some(id => b.ids.indexOf(id) !== -1)) return true;
if (a.urlKey === null || a.urlKey !== b.urlKey) return false;
for (const [k, v] of a.tpl.canon) {
if (k === 'url' || LOCATOR_KEYS.indexOf(k) !== -1 || VOLATILE_KEYS.indexOf(k) !== -1) continue;
if (lenientStatus && k === 'url-status') continue;
if (b.tpl.canon.get(k) !== v) return false;
}
return !lenientStatus || statusCompatible(a, b);
}
function slug(s) {
return normalizeDiacritics(s)
.replace(/[^A-Za-z0-9]+/g, '_')
.replace(/^_+|_+$/g, '')
.toLowerCase()
.slice(0, 40);
}
function uniqueName(base, used) {
let name = base;
for (let n = 2; used.has(name); n++) name = `${base}_${n}`;
used.add(name);
return name;
}
function uniqueAuthorName(base, used) {
if (!used.has(base)) {
used.add(base);
return base;
}
for (let c = 97; c <= 122; c++) {
const cand = base + String.fromCharCode(c);
if (!used.has(cand)) {
used.add(cand);
return cand;
}
}
return uniqueName(base, used);
}
function citeParamsRaw(content) {
const out = new Map();
const t = extractTemplate(content);
if (!t) return out;
const parts = t.parts.slice();
parts[parts.length - 1] = parts[parts.length - 1].replace(/\}\}$/, '');
for (const p of parts.slice(1)) {
const eq = p.indexOf('=');
if (eq === -1) continue;
const k = p.slice(0, eq).replace(/\s+/g, ' ').trim().toLowerCase();
const v = p.slice(eq + 1).replace(/\s+/g, ' ').trim();
if (k && v) out.set(k, v);
}
return out;
}
const ORG_WORD_RE = /\b(?:news|press|staff|team|inc|ltd|llc|corp|company|association|university|institute|department|bureau|agency|committee|council|group|office|services?|society|foundation|ministry|government|times|post|journal|magazine|board|commission|authority|organi[sz]ation|museum|library|project|network)\b/i;
const SURNAME_KEYS = ['last', 'last1', 'surname', 'surname1', 'author-last', 'author-last1', 'author1-last'];
const AUTHOR_KEYS = ['author', 'author1', 'authors'];
function keepNameChar(ch) {
if (/[A-Za-z' -]/.test(ch) || ch === '\u2019') return true;
const c = ch.charCodeAt(0);
if (c < 192 || c === 0xD7 || c === 0xF7) return false;
if (c >= 0x2000 && c <= 0x2BFF) return false;
if (c >= 0x3000 && c <= 0x303F) return false;
return true;
}
function cleanSurname(s) {
const v = s
.replace(/\[\[(?:[^\]|]*\|)?([^\]]*)\]\]/g, '$1')
.replace(/<[^>]*>/g, '')
.replace(/\s+/g, ' ')
.trim();
if (!v || /[\d{}\[\]|=<>\"]/.test(v)) return null;
const out = v.split('').filter(keepNameChar).join('').replace(/\s+/g, ' ').trim();
return out ? out.slice(0, 40) : null;
}
function surnameFromAuthor(value, vancouver) {
const seg = value.split(/\s*(?:;|&|\band\b)\s*/i)[0].trim();
if (!seg || ORG_WORD_RE.test(seg)) return null;
if (vancouver) return cleanSurname(seg.split(/\s*,\s*/)[0].split(' ')[0]);
if (seg.includes(',')) {
const words = seg.split(',')[0].trim().split(' ');
if (words.length > 1 && /^[A-Z]{1,3}\.?$/.test(words[words.length - 1])) words.pop();
return cleanSurname(words.join(' '));
}
const words = seg.split(' ');
if (words.length === 1) return cleanSurname(words[0]);
if (words.length > 3 || words.some(w => /^[A-Z]{2,}$/.test(w))) return null;
return cleanSurname(words[words.length - 1]);
}
function nameFromAuthorYear(content) {
const p = citeParamsRaw(content);
if (!p.size) return null;
let surname = null;
for (const k of SURNAME_KEYS) {
if (p.has(k)) {
surname = cleanSurname(p.get(k));
break;
}
}
if (!surname) {
for (const k of AUTHOR_KEYS) {
if (p.has(k)) {
surname = surnameFromAuthor(p.get(k), false);
break;
}
}
}
if (!surname && p.has('vauthors')) surname = surnameFromAuthor(p.get('vauthors'), true);
if (!surname) return null;
for (const k of ['year', 'date', 'publication-date']) {
const m = p.has(k) && p.get(k).match(/(?:^|\D)(1\d{3}|20\d{2})(?!\d)/);
if (m) return `${surname} ${m[1]}`;
}
return null;
}
function nameFromCiteTitle(content) {
const m = content.match(/\|\s*title\s*=\s*([^|{}\n]+)/i);
if (!m) return null;
const words = normalizeDiacritics(m[1].replace(/<[^>]*>/g, '').trim())
.match(/\b[A-Za-z0-9]+\b/g);
return (words && words.length) ? slug(words.slice(0, 3).join('_')) : null;
}
function nameFromUrl(content) {
const m = content.match(/https?:\/\/([^\s/<>\"]+)/i);
if (!m) return null;
let host = m[1].replace(/^www\./, '');
if (/^(web\.)?archive\.org/.test(host)) {
const arc = content.match(
/archive\.org\/web\/\d+\/https?:\/\/(?:www\.)?([^\s/<>\"]+)/i
);
if (arc) host = arc[1];
}
const domain = host.split('/')[0];
return DOMAIN_BLACKLIST.has(domain) ? null : slug(domain);
}
function nameFromText(content) {
const plain = normalizeDiacritics(content
.replace(/<[^>]*>/g, '')
.replace(/\[\[([^|\]]*)[^\]]*\]\]/g, '$1')
.trim());
const words = plain.match(/\b[A-Za-z0-9]+\b/g);
return (words && words.length) ? slug(words.slice(0, 3).join('_')) : null;
}
function generateName(raw, used) {
const content = getContent(raw);
const hasCite = /\{\{\s*(?:cite|citation)/i.test(content);
if (hasCite) {
const authorYear = nameFromAuthorYear(content);
if (authorYear) return uniqueAuthorName(authorYear, used);
}
const candidates = [];
if (hasCite) candidates.push(nameFromCiteTitle(content));
candidates.push(nameFromUrl(content));
if (!hasCite) candidates.push(nameFromCiteTitle(content));
candidates.push(nameFromText(content));
const base = candidates.find(c => c && !NAME_BLACKLIST.has(c)) || 'ref';
return uniqueName(base, used);
}
function plainName(raw) {
const open = (raw.match(/^<ref[^>]*>/i) || [raw])[0];
const m = open.match(/\bname\s*=\s*(?:"([^\"]+)"|'([^']+)'|([^\s"'“”‘’\/>][^\s"'“”‘’\/>]*))/i);
if (!m) return null;
const v = (m[1] || m[2] || m[3] || '').trim();
return v && v.indexOf('"') === -1 ? v : null;
}
function deduplicate(text) {
const tokens = parseRefs(text);
debug('Found', tokens.length, 'reference tags');
const usedNames = new Set();
for (const t of tokens) {
const n = getName(t.raw);
if (n) {
usedNames.add(n);
const s = slug(n);
if (s) usedNames.add(s);
}
}
const replacements = [];
let count = 0;
let partial = 0;
const allUnnamed = [];
const namedTargets = [];
for (const tok of tokens) {
if (tok.selfClose) continue;
if (isEffectivelyNamed(tok.raw)) {
const name = plainName(tok.raw);
if (name && !/\bfollow\s*=/i.test(tok.raw.match(/^<ref[^>]*>/i)[0])) {
namedTargets.push({ name, d: describeRef(tok) });
}
continue;
}
allUnnamed.push(describeRef(tok));
}
const refs = [];
for (const r of allUnnamed) {
const hits = namedTargets.filter(t => t.d.group === r.group && isSubset(r, t.d));
if (hits.length === 0) {
refs.push(r);
continue;
}
hits.sort((x, y) =>
((x.d.norm === r.norm ? 0 : 1) - (y.d.norm === r.norm ? 0 : 1)) ||
(y.d.score - x.d.score) || (x.d.tok.start - y.d.tok.start));
const t = hits[0];
const gAttr = r.group ? ` group="${r.group}"` : '';
debug('Folding unnamed ref into existing named ref', t.name);
replacements.push({
start: r.tok.start,
end: r.tok.end,
text: `<ref name="${t.name}"${gAttr} />`,
});
count++;
if (r.norm !== t.d.norm) partial++;
}
const parent = refs.map((_, i) => i);
const root = i => {
while (parent[i] !== i) {
parent[i] = parent[parent[i]];
i = parent[i];
}
return i;
};
const owner = new Map();
refs.forEach((r, i) => {
const keys = r.ids.map(id => 'i:' + id);
if (r.urlKey !== null) keys.push('u:' + r.urlKey);
if (keys.length === 0) keys.push('n:' + r.norm);
keys.forEach(k => {
const key = r.group + '\u0000' + k;
if (owner.has(key)) parent[root(i)] = root(owner.get(key));
else owner.set(key, i);
});
});
const buckets = new Map();
refs.forEach((r, i) => {
const k = root(i);
if (!buckets.has(k)) buckets.set(k, []);
buckets.get(k).push(r);
});
debug('Reference buckets:', buckets.size);
for (const entries of buckets.values()) {
if (entries.length < 2) continue;
const ordered = entries.slice().sort((a, b) =>
(b.score - a.score) || (b.len - a.len) || (a.tok.start - b.tok.start));
const clusters = [];
for (const e of ordered) {
const home = clusters.find(c => isSubset(e, c[0], true));
if (home) home.push(e);
else clusters.push([e]);
}
for (const cluster of clusters) {
if (cluster.length < 2) continue;
const keeper = cluster[0];
const members = cluster.slice().sort((a, b) => a.tok.start - b.tok.start);
const first = members[0];
const name = generateName(keeper.tok.raw, usedNames);
const group = getGroup(keeper.tok.raw);
const gAttr = group ? ` group="${group}"` : '';
const content = applyRecency(getContent(keeper.tok.raw), keeper, cluster);
debug('Cluster of', cluster.length, '->', name);
replacements.push({
start: first.tok.start,
end: first.tok.end,
text: `<ref name="${name}"${gAttr}>${content}</ref>`,
});
for (const m of members) {
if (m === first) continue;
replacements.push({
start: m.tok.start,
end: m.tok.end,
text: `<ref name="${name}"${gAttr} />`,
});
count++;
}
for (const m of members) {
if (m !== keeper && m.norm !== keeper.norm) partial++;
}
}
}
if (count === 0) return { text, count: 0, partial: 0 };
replacements.sort((a, b) => b.start - a.start);
let result = text;
for (const r of replacements) {
result = result.slice(0, r.start) + r.text + result.slice(r.end);
}
return { text: result, count, partial };
}
function findOrphanReferences(text) {
const tokens = parseRefs(text);
const defined = new Set();
const used = new Set();
for (const t of tokens) {
const name = getName(t.raw);
if (name === null) continue;
const key = name + '\u0000' + (getGroup(t.raw) || '');
if (t.selfClose) used.add(key);
else defined.add(key);
}
const orphans = [];
for (const key of used) {
if (!defined.has(key)) {
const [name, group] = key.split('\u0000');
orphans.push(group ? `${name} (group="${group}")` : name);
}
}
return orphans;
}
function isCurrentRevision() {
const p = new URLSearchParams(window.location.search);
if (p.get('oldid')) {
debug('Editing old revision (oldid=' + p.get('oldid') + '), skipping');
return false;
}
if (p.get('diff')) {
debug('In diff view, skipping');
return false;
}
const warning = document.querySelector('.mw-revision-warning, .mw-editnotice-base');
if (warning && warning.textContent.toLowerCase().includes('old revision')) {
debug('Old revision warning detected, skipping');
return false;
}
return true;
}
function runOnPage(manual = false) {
debug('Starting deduplication process...');
if (!isCurrentRevision()) return;
const ta = document.getElementById('wpTextbox1');
const sum = document.getElementById('wpSummary');
if (!ta || !sum) {
debug('Edit textarea or summary input not found');
if (manual) {
alert('DeduplicateReferences: Could not find the edit textarea or edit summary.');
}
return;
}
// Use MediaWiki's textSelection API so edits are applied to CodeMirror too.
const $ta = $(ta);
const original = $ta.textSelection('getContents');
const { text, count, partial } = deduplicate(original);
debug('Deduplication complete. Count:', count, 'partial:', partial);
if (count === 0) {
if (manual) alert('No duplicate references found to deduplicate.');
return;
}
const beforeOrphans = new Set(findOrphanReferences(original));
const newOrphans = findOrphanReferences(text).filter(o => !beforeOrphans.has(o));
if (newOrphans.length > 0) {
console.error('[DeduplicateReferences] Aborting, would create orphan reference(s):', newOrphans);
alert(
'DeduplicateReferences aborted: this change would leave the following ' +
'named reference(s) invoked but never defined: ' + newOrphans.join(', ') + '. ' +
'No changes were made. Please report this at the script talk page with the article title.'
);
return;
}
// Update the visible editor, whether CodeMirror or the plain textarea is active.
$ta.textSelection('setContents', text);
const plural = count !== 1 ? 's' : '';
const partialNote = partial > 0
? ` (${partial} merged into a more complete citation of the same URL)`
: '';
const msg = `[[WP:NAMEDREFS|Deduplicated]] ${count} reference${plural}${partialNote} using ${SCRIPT_LINK}`;
sum.value = sum.value ? `${sum.value} • ${msg}` : msg;
const minor = document.querySelector('#wpMinoredit, input[name="wpMinoredit"]');
if (minor) minor.checked = true;
const diffBtn = document.querySelector('#wpDiff, input[name="wpDiff"]');
const shownDiff = !!diffBtn;
if (diffBtn) diffBtn.click();
alert(
`Deduplicated ${count} reference${plural}` +
(partial > 0 ? `, ${partial} of them partial matches merged into a fuller citation.` : '.') +
(shownDiff
? ' Review the diff now loading below before publishing.'
: ' Please review the textarea before publishing.')
);
}
function addPortletLink() {
if (document.getElementById('t-dedupe')) return;
mw.loader.using(['mediawiki.util', 'jquery.textSelection']).then(() => {
if (document.getElementById('t-dedupe')) return;
const item = mw.util.addPortletLink(
'p-tb',
'#',
'Deduplicate references',
't-dedupe',
'Merge duplicate references in the edit box'
);
if (!item) {
debug('Tools portlet not found, link not added');
return;
}
const link = item.querySelector('a');
if (!link) {
debug('Could not find portlet anchor');
return;
}
link.addEventListener('click', e => {
e.preventDefault();
debug('Portlet clicked');
if (!isCurrentRevision()) {
alert('Deduplication only works when editing the current version of an article.');
return;
}
runOnPage(true);
});
debug('Added deduplication portlet link');
}).catch(error => {
console.error('[DeduplicateReferences] Could not load required MediaWiki modules:', error);
});
}
function whenLoaded(fn) {
if (document.readyState === 'complete') {
fn();
} else {
window.addEventListener('load', fn);
}
}
function init() {
if (typeof mw === 'undefined') {
console.log('DeduplicateReferences: MediaWiki not found, aborting.');
return;
}
const wgAction = mw.config.get('wgAction');
const wgNs = mw.config.get('wgNamespaceNumber');
const shouldRun = (wgAction === 'edit' || wgAction === 'submit') &&
ALLOWED_NAMESPACES.has(wgNs);
if (shouldRun) {
debug('Article/draft edit page detected, setting up deduplication...');
whenLoaded(addPortletLink);
} else {
debug('Not on article/draft edit page - action:', wgAction, 'namespace:', wgNs);
}
}
init();
})();