跳转到内容

模組:Graph/DataProcess/Source

被永久保护的模块
维基百科,自由的百科全书
--Module:Graph/DataProcess/Source
--處理與資料來源相關的工作 (及其依賴函數和提供[[Module:Graph/DataProcess]]工具支援)
local p = {}

local WHITELIST_PAGE = "Module:Graph/APIWhiteList.json"
local cachedApiWhitelist = nil

local function parseBool(val, defaultVal)
	if val == nil or val == "" then return defaultVal end
	if val == false or val == 0 or val == "0" then return false end
	local s = string.lower(mw.text.trim(tostring(val)))
	if s == "false" or s == "n" or s == "no" or s == "0" then return false end
	return true
end

-- 遮罩保護 MediaWiki 繁簡轉換與標籤之 Strip Marker,避免內部雙引號破壞 JSON 解析或被 encode 損毀
local function maskStripMarkers(s)
	if type(s) ~= "string" or not string.find(s, "\127", 1, true) then return s end
	s = string.gsub(s, "\127'\"`UNIQ", "__MW_SM_PRE__UNIQ")
	s = string.gsub(s, "QINU`\"'\127", "QINU__MW_SM_SUF__")
	return s
end

local function numericArray(csv)
	if not csv or csv == "" then return nil end
	local list = mw.text.split(tostring(csv), "%s*,%s*")
	local result = { n = #list }
	local isInteger = true
	local hasNumber = false

	for i = 1, #list do
		local item = mw.text.trim(list[i])
		if item == "" then
			result[i] = false
		else
			local num = tonumber(item)
			if not num then return nil end
			hasNumber = true
			result[i] = num
			if isInteger then
				local _, frac = math.modf(num)
				isInteger = (frac == 0.0)
			end
		end
	end
	if not hasNumber then return nil end
	return result, isInteger
end

local function stringArray(text)
	if not text or text == "" then return nil end
	local escaped = mw.ustring.gsub(tostring(text), "\\,", "<COMMA>")
	local list = mw.text.split(escaped, ",", true)
	for i = 1, #list do
		list[i] = mw.ustring.gsub(mw.text.trim(list[i]), "<COMMA>", ",")
	end
	list.n = #list
	return list
end

-- 解析符號與形狀陣列:先檢查標誌性特徵,一般字串直接走 stringArray;含 <svg>、[[...]] 或引號時以 gsub 遮罩切分,零無窮迴圈風險
local function parseShapeArray(text)
	if type(text) == "table" then return text end
	if not text or text == "" then return nil end
	local raw = mw.text.unstripNoWiki(tostring(text))
	if string.find(raw, "&lt;[sS][vV][gG]") then
		raw = mw.text.decode(raw)
	end

	-- 快速通道:若不含 <、[、"、' 等特殊標誌字元,直接走原生 stringArray
	if not string.find(raw, "[<%[\"'\\]") then
		return stringArray(raw)
	end

	local tokens = {}
	local function saveToken(val)
		tokens[#tokens + 1] = val
		return "__ GR_TOK_" .. #tokens .. " __"
	end

	-- 1. 保護完整 <svg>...</svg> 區塊
	local masked = string.gsub(raw, "<[sS][vV][gG][^>]*>.-</[sS][vV][gG]>", saveToken)
	-- 2. 保護 [[File:...|...]] 雙中括號區塊
	masked = string.gsub(masked, "%[%[.-%]%]", saveToken)
	-- 3. 保護雙引號 "..." 與單引號 '...' 區塊 (引號保護法,支援檔名含逗號)
	masked = string.gsub(masked, '"([^"]*)"', saveToken)
	masked = string.gsub(masked, "'([^']*)'", saveToken)

	local list = stringArray(masked)
	if not list then return nil end
	for i = 1, #list do
		local item = list[i]
		if string.find(item, "__ GR_TOK_", 1, true) then
			item = string.gsub(item, "__ GR_TOK_(%d+) __", function(idxStr)
				return tokens[tonumber(idxStr)] or ""
			end)
			list[i] = mw.text.trim(item)
		end
	end
	return list
end

local function isDateArray(arr)
	if not arr or #arr == 0 then return false end
	local validCount = 0
	for i = 1, #arr do
		local s = arr[i]
		if s and s ~= "" then
			if string.match(s, "^%d%d%d%d[%-%/]%d%d?") then
				validCount = validCount + 1
			else
				return false
			end
		end
	end
	return validCount > 0
end

local function deserializeXData(serializedX, xType, xMin, xMax)
	if not serializedX or mw.text.trim(tostring(serializedX)) == "" then
		return { n = 0 }, xType or "number", tonumber(xMin), tonumber(xMax)
	end
	local x
	if not xType or xType == "integer" or xType == "number" then
		local isInteger
		x, isInteger = numericArray(serializedX)
		if x then
			xMin = tonumber(xMin)
			xMax = tonumber(xMax)
			if not xType then
				xType = isInteger and "integer" or "number"
			end
		elseif xType == "integer" or xType == "number" then
			x = stringArray(serializedX)
			xType = "string"
		end
	end
	if not x then
		x = stringArray(serializedX)
		if not xType then
			xType = isDateArray(x) and "date" or "string"
		end
	end
	return x, xType, xMin, xMax
end

local function deserializeYData(serializedYs, yType, yMin, yMax)
	local y = {}
	local areAllInteger = true
	local maxIndex = 0
	for k, _ in pairs(serializedYs) do
		if k > maxIndex then maxIndex = k end
	end

	for yNum = 1, maxIndex do
		local value = serializedYs[yNum]
		if value ~= nil then
			local yValues
			if not yType or yType == "integer" or yType == "number" then
				local isInteger
				yValues, isInteger = numericArray(value)
				if yValues then
					areAllInteger = areAllInteger and isInteger
				else
					return deserializeYData(serializedYs, "string", yMin, yMax)
				end
			end
			if not yValues then yValues = stringArray(value) end
			y[#y + 1] = yValues
		end
	end

	if not yType then
		yType = areAllInteger and "integer" or "number"
	end
	if yType == "integer" or yType == "number" then
		yMin = tonumber(yMin)
		yMax = tonumber(yMax)
	end
	return y, yType, yMin, yMax
end

-- 將 mw.loadJsonData 回傳的唯讀代理表轉為標準 Lua Table (避免 Lua 5.1 中 #t 為 0)
local function toPlainTable(obj)
	if type(obj) ~= "table" then return obj end
	local res = {}
	for k, v in pairs(obj) do
		res[k] = (type(v) == "table") and toPlainTable(v) or v
	end
	return res
end

-- 讀取並快取 Module:Graph/APIWhiteList.json (支援 json 內容模型快取與一般頁面文字解析雙通道)
local function getApiWhitelist()
	if cachedApiWhitelist then return cachedApiWhitelist end

	local data = nil
	if mw.loadJsonData then
		local ok, res = pcall(mw.loadJsonData, WHITELIST_PAGE)
		if ok and type(res) == "table" then
			data = res
		end
	end

	if not data then
		local titleObj = mw.title.new(WHITELIST_PAGE)
		if titleObj then
			local rawContent = titleObj:getContent()
			if rawContent and rawContent ~= "" then
				local okDec, dec = pcall(mw.text.jsonDecode, rawContent)
				if okDec and type(dec) == "table" then
					data = dec
				end
			end
		end
	end

	local exactMap = {}
	local suffixes = {}
	local defaultHost = "wikimedia.org"

	if type(data) == "table" then
		if type(data.wikirestDefaultHost) == "string" and data.wikirestDefaultHost ~= "" then
			defaultHost = string.lower(mw.text.trim(data.wikirestDefaultHost))
		end
		if type(data.exactHosts) == "table" then
			local i = 1
			while data.exactHosts[i] ~= nil do
				local h = string.lower(mw.text.trim(tostring(data.exactHosts[i])))
				if h ~= "" then exactMap[h] = true end
				i = i + 1
			end
		end
		if type(data.hostSuffixes) == "table" then
			local i = 1
			while data.hostSuffixes[i] ~= nil do
				local s = string.lower(mw.text.trim(tostring(data.hostSuffixes[i])))
				if s ~= "" then
					if string.sub(s, 1, 1) ~= "." then s = "." .. s end
					suffixes[#suffixes + 1] = s
				end
				i = i + 1
			end
		end
	else
		-- 預設安全保底白名單 (避免 JSON 頁面尚未建立前全站報錯)
		exactMap = { ["wikimedia.org"] = true, ["wikidata.org"] = true, ["mediawiki.org"] = true, ["toolforge.org"] = true }
		suffixes = { ".wikimedia.org", ".wikipedia.org", ".wikidata.org", ".mediawiki.org", ".toolforge.org", ".wmcloud.org" }
	end

	cachedApiWhitelist = {
		exactMap = exactMap,
		suffixes = suffixes,
		defaultHost = defaultHost
	}
	return cachedApiWhitelist
end

-- 取得當前維基站點的主機名稱 (供 wikiraw:/// 與 wikiapi:/// 三斜線本地協議使用)
local function getLocalWikiHost()
	if mw and mw.site and type(mw.site.server) == "string" then
		local h = string.match(mw.site.server, "//([%w%-%.]+)")
		if h and h ~= "" then return string.lower(h) end
	end
	return "zh.wikipedia.org"
end

-- 清理表頭文字以產生純文字鍵名 (移除 <ref>、HTML 標籤、提取 [[連結|顯示名]] 之顯示名)
local function cleanHeaderName(rawHeader)
	if type(rawHeader) ~= "string" then return "" end
	local s = rawHeader
	s = string.gsub(s, "__MW_SM_PRE__UNIQ.-QINU__MW_SM_SUF__", "")
	s = string.gsub(s, "\127'\"`UNIQ.-QINU`\"'\127", "")
	s = string.gsub(s, "<!%-%-.-%-%->", "")
	s = string.gsub(s, "<[rR][eE][fF][^>]*>.-</[rR][eE][fF]>", "")
	s = string.gsub(s, "<[rR][eE][fF][^/>]*/%s*>", "")
	s = string.gsub(s, "%[%[([^%[%]|]+)|([^%[%]]+)%]%]", "%2")
	s = string.gsub(s, "%[%[([^%[%]]+)%]%]", "%1")
	s = string.gsub(s, "<[^>]+>", "")
	s = string.gsub(s, "&nbsp;", " ")
	s = string.gsub(s, "&#160;", " ")
	return mw.text.trim(s)
end

-- 驗證並正規化動態 API 網址 (嚴格依據 Module:Graph/APIWhiteList.json 進行 HTTPS 與網域白名單檢查)
local function normalizeAndValidateApiUrl(rawUrl)
	if type(rawUrl) ~= "string" then return nil end
	local u = mw.text.trim(rawUrl)
	-- 阻擋控制字元、HTML 標籤字元、引號與路徑穿越 (..)
	if u == "" or string.find(u, "[%c<>\"'\\]") or string.find(u, "%.%.", 1, true) then
		return nil
	end

	local wl = getApiWhitelist()
	local fullUrl = u

	-- 1. 支援 wikiraw://、wikiapi:// 與 wikirest:// 協議
	if string.match(fullUrl, "^wikiraw:///") then
		local pageTitle = string.gsub(string.gsub(fullUrl, "^wikiraw:///", ""), " ", "_")
		fullUrl = "https://" .. getLocalWikiHost() .. "/w/index.php?title=" .. pageTitle .. "&action=raw"
	elseif string.match(fullUrl, "^wikiraw://") then
		local hostPart, pageTitle = string.match(fullUrl, "^wikiraw://([%w%-%.]+)/(.+)$")
		if not hostPart or not pageTitle then return nil end
		pageTitle = string.gsub(pageTitle, " ", "_")
		fullUrl = "https://" .. hostPart .. "/w/index.php?title=" .. pageTitle .. "&action=raw"
	elseif string.match(fullUrl, "^wikiapi:///") then
		local queryPart = string.gsub(string.gsub(fullUrl, "^wikiapi:///?", ""), " ", "%%20")
		fullUrl = "https://" .. getLocalWikiHost() .. "/w/api.php?" .. queryPart
	elseif string.match(fullUrl, "^wikirest:///") then
		fullUrl = "https://" .. wl.defaultHost .. "/" .. string.gsub(string.gsub(fullUrl, "^wikirest:///", ""), " ", "%%20")
	elseif string.match(fullUrl, "^wikirest://") then
		fullUrl = "https://" .. string.gsub(string.gsub(fullUrl, "^wikirest://", ""), " ", "%%20")
	elseif string.sub(fullUrl, 1, 2) == "//" then
		fullUrl = "https:" .. string.gsub(fullUrl, " ", "%%20")
	else
		fullUrl = string.gsub(fullUrl, " ", "%%20")
	end

	-- 2. 嚴格禁止明文 http://,僅允許 https:// 且網域須位於白名單內
	local host, path = string.match(fullUrl, "^https://([%w%-%.]+)(/.*)$")
	if not host or not path then return nil end
	host = string.lower(host)

	local isAllowedHost = (wl.exactMap[host] == true)
	if not isAllowedHost then
		for i = 1, #wl.suffixes do
			local suf = wl.suffixes[i]
			local sLen = #suf
			if #host > sLen and string.sub(host, -sLen) == suf then
				isAllowedHost = true
				break
			end
		end
	end

	if not isAllowedHost then return nil end

	return fullUrl
end

-- 針對非維基白名單之第三方外部 API (如 NASA、NOAA):驗證為語法安全之外部 HTTPS 網址,供 gr-async-chart-unsupported 容器封裝
local function normalizeUnsupportedExternalUrl(rawUrl)
	if type(rawUrl) ~= "string" then return nil end
	local u = mw.text.trim(rawUrl)
	if u == "" or string.find(u, "[%c<>\"'\\]") or string.find(u, "%.%.", 1, true) then
		return nil
	end
	u = string.gsub(u, " ", "%%20")
	if string.sub(u, 1, 2) == "//" then
		u = "https:" .. u
	elseif string.match(u, "^http://") then
		u = "https://" .. string.sub(u, 8)
	end
	local host, path = string.match(u, "^https://([%w%-%.]+)(/.*)$")
	if not host or not path then return nil end
	host = string.lower(host)
	-- 阻擋 localhost 與純 IPv4 內網位址,僅允許合法外部網域名稱
	if host == "localhost" or string.match(host, "^%d+%.%d+%.%d+%.%d+$") or not string.find(host, "%.", 1, true) then
		return nil
	end
	return u
end

-- 將模板參數跳脫模式之表格語法 ({{(!}}, {{!}}, {{!!}}, {{!-}}, {{!)}} 與 {{=}}) 正規化為標準 Wikitext 表格語法
local function normalizeTemplateTableSyntax(s)
	if type(s) ~= "string" or s == "" then return "" end
	s = string.gsub(s, "{{%s*%(%!%s*}}", "{|")
	s = string.gsub(s, "{{%s*%!%)%s*}}", "|}")
	s = string.gsub(s, "{{%s*%!%-%s*}}", "|-")
	s = string.gsub(s, "{{%s*%!%!%s*}}", "||")
	s = string.gsub(s, "{{%s*%!%s*}}", "|")
	s = string.gsub(s, "{{%s*=%s*}}", "=")
	return s
end

-- 清理資料儲存格內容,若為純數值 (含千分位逗號) 則自動轉為 number,否則保留原始字串 (如 [[User:xxx]])
local function parseWikitableCellValue(rawCell)
	if type(rawCell) ~= "string" then return "" end
	local s = rawCell
	s = string.gsub(s, "<!%-%-.-%-%->", "")
	s = string.gsub(s, "<[rR][eE][fF][^>]*>.-</[rR][eE][fF]>", "")
	s = string.gsub(s, "<[rR][eE][fF][^/>]*/%s*>", "")
	s = string.gsub(s, "&nbsp;", " ")
	s = string.gsub(s, "&#160;", " ")
	s = mw.text.trim(s)
	if s == "" then return "" end

	-- 嘗試解析為數值 (先移除 Strip Marker 與千分位逗號進行檢測)
	local numCand = string.gsub(s, "__MW_SM_PRE__UNIQ.-QINU__MW_SM_SUF__", "")
	numCand = string.gsub(numCand, "\127'\"`UNIQ.-QINU`\"'\127", "")
	numCand = mw.text.trim(numCand)
	local noComma = string.gsub(numCand, ",", "")
	local numV = tonumber(noComma)
	if numV ~= nil then
		return numV
	end
	return s
end

-- 深度感知分離單一儲存格之「HTML 屬性」與「內容」 (例如 style="color:red" | [[連結|文字]])
local function splitCellAttrsAndContent(rawCell)
	local i, len = 1, #rawCell
	local linkDepth, tmplDepth, tagDepth = 0, 0, 0
	local quoteChar = nil

	while i <= len do
		local two = string.sub(rawCell, i, i + 1)
		local ch = string.sub(rawCell, i, i)

		if quoteChar then
			if ch == quoteChar then quoteChar = nil end
			i = i + 1
		else
			-- 儲存格屬性不可跨行,若在遇到單一 | 前先遇到換行,代表此格無屬性前綴
			if ch == "\n" then
				return "", rawCell
			elseif ch == '"' or ch == "'" then
				quoteChar = ch
				i = i + 1
			elseif two == "[[" then
				linkDepth = linkDepth + 1
				i = i + 2
			elseif two == "]]" and linkDepth > 0 then
				linkDepth = linkDepth - 1
				i = i + 2
			elseif two == "{{" then
				tmplDepth = tmplDepth + 1
				i = i + 2
			elseif two == "}}" and tmplDepth > 0 then
				tmplDepth = tmplDepth - 1
				i = i + 2
			elseif ch == "<" then
				tagDepth = tagDepth + 1
				i = i + 1
			elseif ch == ">" and tagDepth > 0 then
				tagDepth = tagDepth - 1
				i = i + 1
			elseif ch == "|" and two ~= "||" and linkDepth == 0 and tmplDepth == 0 and tagDepth == 0 then
				return string.sub(rawCell, 1, i - 1), string.sub(rawCell, i + 1)
			else
				i = i + 1
			end
		end
	end
	return "", rawCell
end

-- 深度感知切分同行儲存格:在 [[...]]、{{...}}、<...> 深度為 0 時,以 || (或表頭列的 !!) 切分
local function splitInlineWikitableCells(lineBody, isHeaderLine)
	local cells = {}
	local cur = {}
	local i, len = 1, #lineBody
	local linkDepth, tmplDepth, tagDepth = 0, 0, 0

	while i <= len do
		local two = string.sub(lineBody, i, i + 1)
		local ch = string.sub(lineBody, i, i)

		if two == "[[" then
			linkDepth = linkDepth + 1
			cur[#cur + 1] = "[["
			i = i + 2
		elseif two == "]]" and linkDepth > 0 then
			linkDepth = linkDepth - 1
			cur[#cur + 1] = "]]"
			i = i + 2
		elseif two == "{{" then
			tmplDepth = tmplDepth + 1
			cur[#cur + 1] = "{{"
			i = i + 2
		elseif two == "}}" and tmplDepth > 0 then
			tmplDepth = tmplDepth - 1
			cur[#cur + 1] = "}}"
			i = i + 2
		elseif ch == "<" then
			tagDepth = tagDepth + 1
			cur[#cur + 1] = ch
			i = i + 1
		elseif ch == ">" and tagDepth > 0 then
			tagDepth = tagDepth - 1
			cur[#cur + 1] = ch
			i = i + 1
		elseif linkDepth == 0 and tmplDepth == 0 and tagDepth == 0 and (two == "||" or (isHeaderLine and two == "!!")) then
			cells[#cells + 1] = table.concat(cur)
			cur = {}
			i = i + 2
		else
			cur[#cur + 1] = ch
			i = i + 1
		end
	end
	cells[#cells + 1] = table.concat(cur)
	return cells
end

-- 解析單一標準化後的 {| ... |} 表格區塊為二維展開網格與紀錄陣列
local function parseSingleWikitable(tableText)
	local normText = normalizeTemplateTableSyntax(tableText)
	normText = string.gsub(normText, "\r\n", "\n")
	normText = string.gsub(normText, "\r", "\n")

	local lines = mw.text.split(normText, "\n", true)
	local rawRows = {}
	local curRow = nil
	local caption = nil
	local inTable = false

	local function ensureRow()
		if not curRow then
			curRow = { cells = {}, hasHeaderCell = false, hasDataCell = false }
			rawRows[#rawRows + 1] = curRow
		end
	end

	for lIdx = 1, #lines do
		local line = mw.text.trim(lines[lIdx])
		if not inTable then
			if string.sub(line, 1, 2) == "{|" then
				inTable = true
			end
		else
			if string.sub(line, 1, 2) == "|}" then
				break
			elseif string.sub(line, 1, 2) == "|+" then
				local capBody = string.sub(line, 3)
				local _, capContent = splitCellAttrsAndContent(capBody)
				caption = cleanHeaderName(capContent)
			elseif string.sub(line, 1, 2) == "|-" then
				if curRow and #curRow.cells > 0 then
					curRow = nil
				end
			elseif string.sub(line, 1, 1) == "!" or string.sub(line, 1, 1) == "|" then
				ensureRow()
				local isHeaderLine = (string.sub(line, 1, 1) == "!")
				local lineBody
				-- 相容行首連續寫 !! 或 || 的情況
				if string.sub(line, 1, 2) == "!!" or string.sub(line, 1, 2) == "||" then
					lineBody = string.sub(line, 3)
				else
					lineBody = string.sub(line, 2)
				end

				if isHeaderLine then
					curRow.hasHeaderCell = true
				else
					curRow.hasDataCell = true
				end

				local inlineList = splitInlineWikitableCells(lineBody, isHeaderLine)
				for cIdx = 1, #inlineList do
					local attrs, content = splitCellAttrsAndContent(inlineList[cIdx])
					local rowspan = tonumber(string.match(attrs, '[rR][oO][wW][sS][pP][aA][nN]%s*=%s*["\']?(%d+)')) or 1
					local colspan = tonumber(string.match(attrs, '[cC][oO][lL][sS][pP][aA][nN]%s*=%s*["\']?(%d+)')) or 1
					local sortVal = string.match(attrs, 'data%-sort%-value%s*=%s*"([^"]*)"')
						or string.match(attrs, "data%-sort%-value%s*=%s*'([^']*)'")
					curRow.cells[#curRow.cells + 1] = {
						raw = content,
						isHeader = isHeaderLine,
						rowspan = math.max(1, math.min(50, rowspan)),
						colspan = math.max(1, math.min(50, colspan)),
						sortVal = sortVal
					}
				end
			elseif curRow and #curRow.cells > 0 and line ~= "" then
				-- 跨行儲存格內容續接
				local lastCell = curRow.cells[#curRow.cells]
				lastCell.raw = lastCell.raw .. "\n" .. line
			end
		end
	end

	if #rawRows == 0 then return nil end

	-- 將含有 rowspan / colspan 的列展開為對齊的二維矩陣 grid[r][c]
	local grid = {}
	local rowMeta = {}
	local maxCols = 0
	for r = 1, #rawRows do
		grid[r] = grid[r] or {}
		rowMeta[r] = { hasHeader = rawRows[r].hasHeaderCell, hasData = rawRows[r].hasDataCell }
		local colPtr = 1
		for _, cell in ipairs(rawRows[r].cells) do
			while grid[r][colPtr] ~= nil do
				colPtr = colPtr + 1
			end
			for dr = 0, cell.rowspan - 1 do
				local tr = r + dr
				grid[tr] = grid[tr] or {}
				for dc = 0, cell.colspan - 1 do
					local tc = colPtr + dc
					grid[tr][tc] = cell
					if tc > maxCols then maxCols = tc end
				end
			end
			colPtr = colPtr + cell.colspan
		end
	end

	if maxCols == 0 then return nil end

	-- 判定表頭列數:前導含有 ! 的列視為表頭;若第 1 列無 ! 但全為非數值文字且第 2 列含數值,亦視為表頭
	local headerRowCount = 0
	for r = 1, #rawRows do
		if rowMeta[r].hasHeader and not rowMeta[r].hasData then
			headerRowCount = r
		elseif r == 1 and rowMeta[r].hasHeader then
			headerRowCount = 1
			break
		else
			break
		end
	end

	if headerRowCount == 0 and #rawRows >= 2 then
		local r1HasNum, r2HasNum = false, false
		for c = 1, maxCols do
			local c1 = grid[1] and grid[1][c]
			local c2 = grid[2] and grid[2][c]
			if c1 and type(parseWikitableCellValue(c1.raw)) == "number" then r1HasNum = true end
			if c2 and type(parseWikitableCellValue(c2.raw)) == "number" then r2HasNum = true end
		end
		if not r1HasNum and r2HasNum then
			headerRowCount = 1
		end
	end

	local headers = {}
	local rawHeaders = {}
	for c = 1, maxCols do
		local hParts = {}
		local lastRawH = nil
		for r = 1, headerRowCount do
			local cell = grid[r] and grid[r][c]
			if cell then
				local ch = cleanHeaderName(cell.raw)
				if ch ~= "" and (hParts[#hParts] ~= ch) then
					hParts[#hParts + 1] = ch
					lastRawH = mw.text.trim(cell.raw)
				end
			end
		end
		local finalH = (#hParts > 0) and hParts[#hParts] or ("col" .. c)
		headers[c] = finalH
		rawHeaders[c] = lastRawH or finalH
	end

	local records = {}
	local startDataRow = headerRowCount + 1
	for r = startDataRow, #rawRows do
		-- 略過夾在資料中間的純重複表頭列
		if not (rowMeta[r].hasHeader and not rowMeta[r].hasData) then
			local rec = {}
			local hasNonEmpty = false
			for c = 1, maxCols do
				local cell = grid[r] and grid[r][c]
				local val = cell and parseWikitableCellValue(cell.raw) or ""
				if val ~= "" then hasNonEmpty = true end
				local hClean = headers[c]
				local hRaw = rawHeaders[c]
				rec[hClean] = val
				if hRaw and hRaw ~= hClean and rec[hRaw] == nil then
					rec[hRaw] = val
				end
				-- 同步建立底線與空白別名,並支援 1-based 索引供 getRecordField ("0", "1") 讀取
				local spaceAlias = string.gsub(hClean, "_", " ")
				if spaceAlias ~= hClean and rec[spaceAlias] == nil then rec[spaceAlias] = val end
				local underAlias = string.gsub(hClean, " ", "_")
				if underAlias ~= hClean and rec[underAlias] == nil then rec[underAlias] = val end
			end
			if hasNonEmpty then
				records[#records + 1] = rec
			end
		end
	end

	if #records == 0 then return nil end
	records._headers = headers
	records._caption = caption
	return records
end

-- 從頁面或段落文字中掃描所有頂層 {| ... |} 表格並依條件或欄位智慧挑選最佳表格
local function parseWikitableFromContent(content, selector, hintFields, wf)
	if type(content) ~= "string" or content == "" then return nil end
	local text = normalizeTemplateTableSyntax(content)
	if not string.find(text, "{|", 1, true) then return nil end

	-- 掃描並提取所有頂層 {| ... |} 區塊及其所屬章節標題
	local candidates = {}
	local i, len = 1, #text
	local depth = 0
	local startIdx = nil
	local currentSection = ""

	while i <= len do
		local two = string.sub(text, i, i + 1)
		if depth == 0 and (i == 1 or string.byte(text, i - 1) == 10) and string.sub(text, i, i + 1) == "==" then
			local lineEnd = string.find(text, "\n", i, true) or (len + 1)
			local headingLine = string.sub(text, i, lineEnd - 1)
			local hText = string.match(headingLine, "^=+%s*(.-)%s*=+%s*$")
			if hText and hText ~= "" then
				currentSection = cleanHeaderName(hText)
			end
			i = lineEnd
		elseif two == "{|" then
			if depth == 0 then startIdx = i end
			depth = depth + 1
			i = i + 2
		elseif two == "|}" and depth > 0 then
			depth = depth - 1
			if depth == 0 and startIdx then
				local block = string.sub(text, startIdx, i + 1)
				local firstLine = string.match(block, "^([^\r\n]+)") or ""
				local isWikitableClass = string.find(string.lower(firstLine), "wikitable", 1, true) ~= nil
				local tableId = string.match(firstLine, '[iI][dD]%s*=%s*"([^"]+)"')
					or string.match(firstLine, "[iI][dD]%s*=%s*'([^']+)'")
				candidates[#candidates + 1] = {
					idx = #candidates + 1,
					raw = block,
					isWikitable = isWikitableClass,
					id = tableId,
					section = currentSection
				}
				startIdx = nil
			end
			i = i + 2
		else
			i = i + 1
		end
	end

	if #candidates == 0 then return nil end

	-- 對單一候選表格進行按需局部 preprocess 與解析
	local function decodeCandidate(cand)
		if cand._decoded ~= nil then return cand._decoded or nil end
		local rawBlock = cand.raw
		if wf and wf.preprocess and string.find(rawBlock, "{{", 1, true) then
			local okPre, preRes = pcall(function() return wf:preprocess(rawBlock) end)
			if okPre and type(preRes) == "string" and preRes ~= "" then
				rawBlock = preRes
			end
		end
		local recs = parseSingleWikitable(rawBlock)
		cand._decoded = recs or false
		return recs
	end

	-- 1. 若有明確指定選擇器 (如 #1, #2 或章節/標題/ID 名稱)
	if selector and selector ~= "" then
		local selTrim = mw.text.trim(tostring(selector))
		selTrim = string.gsub(selTrim, "^#", "")
		local selNum = tonumber(selTrim)
		if selNum and selNum >= 1 then
			-- 優先取第 N 個 class="wikitable" 的表格;若不足則取第 N 個任意表格
			local wtList = {}
			for _, c in ipairs(candidates) do
				if c.isWikitable then wtList[#wtList + 1] = c end
			end
			local target = wtList[selNum] or candidates[selNum]
			if target then
				local recs = decodeCandidate(target)
				if recs then return recs end
			end
		else
			for _, c in ipairs(candidates) do
				if (c.id and c.id == selTrim) or (c.section ~= "" and string.find(c.section, selTrim, 1, true)) then
					local recs = decodeCandidate(c)
					if recs then return recs end
				end
			end
			for _, c in ipairs(candidates) do
				local recs = decodeCandidate(c)
				if recs and recs._caption and string.find(recs._caption, selTrim, 1, true) then
					return recs
				end
			end
		end
	end

	-- 2. 收集期望匹配的欄位名 (供多表格智慧嗅探)
	local wantedFields = {}
	if type(hintFields) == "table" then
		local checkKeys = { "xField", "xfield", "yField", "yfield", "zField", "zfield", "groupField", "groupfield", "seriesField" }
		for _, k in ipairs(checkKeys) do
			if type(hintFields[k]) == "string" and hintFields[k] ~= "" then
				wantedFields[#wantedFields + 1] = cleanHeaderName(hintFields[k])
			end
		end
		local fIdx = 1
		while hintFields["y" .. fIdx .. "Field"] or hintFields["y" .. fIdx .. "field"] do
			wantedFields[#wantedFields + 1] = cleanHeaderName(hintFields["y" .. fIdx .. "Field"] or hintFields["y" .. fIdx .. "field"])
			fIdx = fIdx + 1
		end
	end

	if #wantedFields > 0 then
		local bestRecs = nil
		local bestScore = 0
		for _, c in ipairs(candidates) do
			local recs = decodeCandidate(c)
			if recs and recs[1] then
				local score = 0
				for _, wfName in ipairs(wantedFields) do
					if recs[1][wfName] ~= nil then
						score = score + 10
					end
				end
				if score > 0 and c.isWikitable then
					score = score + 1
				end
				if score > bestScore then
					bestScore = score
					bestRecs = recs
				end
			end
		end
		if bestRecs then return bestRecs end
	end

	-- 3. 預設退回:優先取第一個帶有 class="wikitable" 且有資料的表格,其次取第一個有效表格
	for _, c in ipairs(candidates) do
		if c.isWikitable then
			local recs = decodeCandidate(c)
			if recs then return recs end
		end
	end
	for _, c in ipairs(candidates) do
		local recs = decodeCandidate(c)
		if recs then return recs end
	end

	return nil
end

-- 從巢狀或 Metatable 代理表中提取資料陣列 (支援 formatSpec.property == "fields" 與欄位別名映射)
local function extractRecordList(rawObj, formatSpec)
	if type(rawObj) ~= "table" then return nil end
	local reqProp = type(formatSpec) == "table" and formatSpec.property or nil

	-- 若 Vega 明確要求讀取 schema.fields (如 Template:Graph:Lines 的 labels 資料源)
	if reqProp == "fields" and type(rawObj.schema) == "table" and type(rawObj.schema.fields) == "table" then
		local outFields = {}
		local fIdx = 1
		while rawObj.schema.fields[fIdx] ~= nil do
			local f = rawObj.schema.fields[fIdx]
			local tVal = f.title
			if type(tVal) == "table" then
				tVal = tVal.zh or tVal["zh-hant"] or tVal["zh-hans"] or tVal.en or next(tVal) and select(2, next(tVal))
			end
			outFields[fIdx] = {
				name = f.name or tostring(fIdx),
				title = tostring(tVal or f.name or fIdx)
			}
			fIdx = fIdx + 1
		end
		return outFields
	end

	if rawObj[1] ~= nil then return rawObj end

	-- 常見的外層鍵名(statistics 為 Template:Unblock-zh backlog statistics/data.json 所用)
	local knownKeys = { "statistics", "records", "rows", "items" }
	for _, k in ipairs(knownKeys) do
		local v = rawObj[k]
		if type(v) == "table" and v[1] ~= nil and type(v[1]) == "table" then
			return v
		end
	end

	if type(rawObj.data) == "table" then
		-- 檢查是否為 Commons Tabular Data 格式 (含 schema.fields 與二維陣列/字典 data)
		if type(rawObj.schema) == "table" and type(rawObj.schema.fields) == "table" then
			local fields = {}
			local titles = {}
			local fIdx = 1
			while rawObj.schema.fields[fIdx] ~= nil do
				local f = rawObj.schema.fields[fIdx]
				local fName = f.name or tostring(fIdx)
				fields[fIdx] = fName
				local tVal = f.title
				if type(tVal) == "table" then
					tVal = tVal.en or tVal.zh or tVal["zh-hant"]
				end
				if type(tVal) == "string" and tVal ~= "" and tVal ~= fName then
					titles[fIdx] = tVal
				end
				fIdx = fIdx + 1
			end
			local records = {}
			local rIdx = 1
			while rawObj.data[rIdx] ~= nil do
				local row = rawObj.data[rIdx]
				local rec = {}
				for cIdx = 1, #fields do
					local fName = fields[cIdx]
					local val = row[cIdx]
					if val == nil then val = row[fName] end
					rec[fName] = val
					-- 同步建立「底線轉空白」與「空白轉底線」的雙向別名 (如 New_Zealand <-> New Zealand)
					local spaceAlias = string.gsub(fName, "_", " ")
					if spaceAlias ~= fName and rec[spaceAlias] == nil then
						rec[spaceAlias] = val
					end
					local underAlias = string.gsub(fName, " ", "_")
					if underAlias ~= fName and rec[underAlias] == nil then
						rec[underAlias] = val
					end
					-- 同步建立 schema.fields[].title 別名
					local tAlias = titles[cIdx]
					if tAlias and rec[tAlias] == nil then
						rec[tAlias] = val
					end
				end
				records[rIdx] = rec
				rIdx = rIdx + 1
			end
			return records
		end
		if rawObj.data[1] ~= nil then
			if type(rawObj.data[1]) == "table" and rawObj.data[1].values then
				return rawObj.data[1].values
			end
			return rawObj.data
		end
	end
	if type(rawObj.values) == "table" and rawObj.values[1] ~= nil then
		return rawObj.values
	end
	if type(rawObj.series) == "table" and rawObj.series[1] ~= nil then
		return rawObj.series
	end

	-- 通用備案:依鍵名排序,取第一個「陣列且元素為記錄表」的欄位
	local keys = {}
	for k, v in pairs(rawObj) do
		if type(k) == "string" and type(v) == "table" and v[1] ~= nil and type(v[1]) == "table" then
			keys[#keys + 1] = k
		end
	end
	table.sort(keys)
	if #keys > 0 then
		return rawObj[keys[1]]
	end

	return nil
end

-- 5 階段外部資料路由器 (Data Router)
local function fetchExternalData(rawUrl, frame, formatSpec, hintFields)
	local DataProcesser = require("Module:Graph/DataProcess")
	if not rawUrl or mw.text.trim(tostring(rawUrl)) == "" then
		return nil, false
	end
	local url = mw.text.trim(tostring(rawUrl))
	local wf = (frame and frame.preprocess) and frame or mw.getCurrentFrame()
	local explicitSelector = (type(formatSpec) == "table" and (formatSpec.table or formatSpec.selector or formatSpec.caption or formatSpec.section or formatSpec.id))
		or (type(hintFields) == "table" and (hintFields.tableIndex or hintFields.tableindex or hintFields.tableSelector or hintFields.tableselector))
		or nil

	-- 【路由 A-Inline】若直接傳入行內 Wikitable ({|...|}) 或模板跳脫表格 ({{(!}}...{{!)}}),直接解析無須讀取頁面
	if string.find(url, "{|", 1, true) or string.find(url, "{{(!", 1, true) then
		local wtRecords = parseWikitableFromContent(url, explicitSelector, hintFields, wf)
		if wtRecords then
			return wtRecords, false
		end
		return nil, true
	end

	-- 【路由 A0】Wikidata SPARQL 雙通道:無論寫 wikidatasparql:// 或 https://query.wikidata.org/sparql,
	-- 一律優先由後端 Lua (Module:Graph/Wikidata) 執行並輸出靜態 SVG;僅當查詢超出 Lua 能力時才轉交前端 "async"
	if string.match(url, "^wikidatasparql:/")
		or string.match(url, "^https?://query%.wikidata%.org/")
		or string.match(url, "^//query%.wikidata%.org/") then
		local okW, Wikidata = pcall(require, "Module:Graph/Wikidata")
		if okW and type(Wikidata) == "table" then
			if Wikidata.query then
				local records, shouldAbort = Wikidata.query(url)
				if not shouldAbort and type(records) == "table" and #records > 0 then
					return records, false
				end
			end
			if Wikidata.toStandardSparqlEndpointUrl then
				local fallbackSparqlUrl = Wikidata.toStandardSparqlEndpointUrl(url)
				if fallbackSparqlUrl then
					return fallbackSparqlUrl, "async"
				end
			end
		end
		return nil, true
	end

	-- 【路由 A】wikirest:// 或外部動態 HTTP API:
	-- 1. 通過維基官方白名單者標記為 "async" (輸出 gr-async-chart)
	-- 2. 非維基白名單之第三方外部 API (如 NASA、NOAA) 標記為 "async_unsupported" (輸出 gr-async-chart-unsupported)
	if string.match(url, "^wikirest://") or string.match(url, "^https?://") or string.match(url, "^//") then
		local validApiUrl = normalizeAndValidateApiUrl(url)
		if validApiUrl then
			return validApiUrl, "async"
		end
		local thirdPartyUrl = normalizeUnsupportedExternalUrl(url)
		if thirdPartyUrl then
			return thirdPartyUrl, "async_unsupported"
		end
		return nil, true
	end

	-- 清理 wikitable:///、wikiraw:///、tabular:///、map:/// 前綴並還原 URL 編碼路徑
	local pageName = url
	local isTabularProtocol = false
	local isWikitableProtocol = false
	if string.match(pageName, "^wikitable:/") then
		pageName = string.gsub(pageName, "^wikitable:///?", "")
		pageName = mw.uri.decode(pageName, "PATH")
		isWikitableProtocol = true
	elseif string.sub(pageName, 1, 9) == "wikiraw:/" then
		pageName = string.gsub(pageName, "^wikiraw:///?", "")
		pageName = mw.uri.decode(pageName, "PATH")
	elseif string.match(pageName, "^tabular:/") or string.match(pageName, "^map:/") then
		pageName = string.gsub(pageName, "^tabular:///?", "")
		pageName = string.gsub(pageName, "^map:///?", "")
		pageName = mw.uri.decode(pageName, "PATH")
		isTabularProtocol = true
	end
	pageName = mw.text.trim(pageName)

	-- 分離頁面名稱後的 #選擇器 (例如 "Wikipedia:用户脚本#2" 或 "#1")
	local hashPage, hashSel = string.match(pageName, "^(.-)#(.+)$")
	if hashSel then
		pageName = mw.text.trim(hashPage)
		explicitSelector = explicitSelector or mw.text.trim(hashSel)
	end

	-- 若為 wikitable:/// (或 self) 未指定頁面名,預設讀取當前頁面自身之 Wikitable
	if (pageName == "" or string.lower(pageName) == "self" or string.lower(pageName) == "this") and mw.title.getCurrentTitle then
		local curTitle = mw.title.getCurrentTitle()
		if curTitle and curTitle.prefixedText ~= "" then
			pageName = curTitle.prefixedText
			isWikitableProtocol = true
		end
	end

	-- 【路由 B】維基共享資源 Data: 命名空間 (mw.ext.data.get,若失敗則繼續往路由 D/E 尋找本地 Data: 頁面)
	if not isWikitableProtocol and (isTabularProtocol or string.match(pageName, "^[Cc]ommons:[Dd]ata:") or string.match(pageName, "^[Dd]ata:")) then
		local cleanDataPage = string.gsub(pageName, "^[Cc]ommons:[Dd]ata:", "")
		cleanDataPage = string.gsub(cleanDataPage, "^[Dd]ata:", "")
		local spaceDataPage = string.gsub(cleanDataPage, "_", " ")
		if mw.ext and mw.ext.data and mw.ext.data.get then
			local ok, tabData = pcall(mw.ext.data.get, spaceDataPage)
			if not ok or type(tabData) ~= "table" then
				ok, tabData = pcall(mw.ext.data.get, cleanDataPage)
			end
			if ok and type(tabData) == "table" then
				local list = extractRecordList(tabData, formatSpec)
				if list then return list, false end
			end
		end
		pageName = "Data:" .. cleanDataPage
	end

	-- 【路由 C】Module: 命名空間 (mw.loadData)
	if not isWikitableProtocol and (string.match(pageName, "^[Mm]odule:") or string.match(pageName, "^模組:") or string.match(pageName, "^模块:")) then
		local ok, modData = pcall(mw.loadData, pageName)
		if not ok or type(modData) ~= "table" then
			ok, modData = pcall(require, pageName)
		end
		if ok and type(modData) == "table" then
			local list = extractRecordList(modData, formatSpec)
			if list then return list, false end
		end
		return nil, true
	end

	-- 取得頁面 Title 物件,檢查【路由 D】與【路由 E】
	local titleObj = mw.title.new(pageName)
	if not titleObj then return nil, true end

	-- 【路由 D】檢查 contentModel === 'json',優先使用 mw.loadJsonData 快取通道
	if not isWikitableProtocol and titleObj.contentModel == "json" and mw.loadJsonData then
		local ok, jsonData = pcall(mw.loadJsonData, pageName)
		if ok and type(jsonData) == "table" then
			local list = extractRecordList(jsonData, formatSpec)
			if list then return toPlainTable(list), false end
		end
	end

	-- 【路由 E】一般維基頁面:支援 Wikitable、JSON 或 CSV
	local content = titleObj:getContent()
	if not content or content == "" then return nil, true end

	content = string.gsub(content, "<noinclude>.-</noinclude>", "")
	content = string.gsub(content, "</?includeonly>", "")
	content = string.gsub(content, "</?onlyinclude>", "")

	local isWikitableFormat = isWikitableProtocol
		or (type(formatSpec) == "table" and string.lower(tostring(formatSpec.type or "")) == "wikitable")
		or string.find(content, "{|", 1, true) ~= nil
		or string.find(content, "{{(!", 1, true) ~= nil

	-- E1. 若頁面含有 Wikitable ({| 或 {{(!}}),優先以局部按需展開方式解析表格 (避免對整篇長條目執行 preprocess)
	if isWikitableFormat then
		local wtList = parseWikitableFromContent(content, explicitSelector, hintFields, wf)
		if wtList then return wtList, false end
		if isWikitableProtocol then return nil, true end
	end

	-- E2. 一般 JSON 或 CSV 資料頁:若含模板則展開後解析
	if wf and wf.preprocess and string.find(content, "{{", 1, true) then
		local okPre, preRes = pcall(function() return wf:preprocess(content) end)
		if okPre and preRes then content = preRes end
	end

	-- 若展開模板後才浮現 {| 表格,再次嘗試解析為 Wikitable
	if string.find(content, "{|", 1, true) or string.find(content, "{{(!", 1, true) then
		local wtList = parseWikitableFromContent(content, explicitSelector, hintFields, nil)
		if wtList then return wtList, false end
	end

	local isCsvFormat = (type(formatSpec) == "table" and string.lower(tostring(formatSpec.type or "")) == "csv")
		or string.match(pageName, "[%-%.]csv$") ~= nil

	if isCsvFormat then
		local csvList = DataProcesser.parseCSV(mw.text.trim(content))
		if csvList then return csvList, false end
	end
	local cleanJson = DataProcesser.stripJsonComments(mw.text.trim(content))
	local okDec, decoded = pcall(mw.text.jsonDecode, cleanJson, mw.text.JSON_TRY_FIXING)
	if okDec and type(decoded) == "table" then
		local list = extractRecordList(decoded, formatSpec)
		if list then return list, false end
	end

	-- 若未標記 csv 但 JSON 解析失敗,嘗試當作 CSV 解析
	if not isCsvFormat then
		local csvList = DataProcesser.parseCSV(mw.text.trim(content))
		if csvList then return csvList, false end
	end

	return nil, true
end

p.WHITELIST_PAGE = WHITELIST_PAGE
p.parseBool = parseBool
p.maskStripMarkers = maskStripMarkers
p.numericArray = numericArray
p.stringArray = stringArray
p.parseShapeArray = parseShapeArray
p.isDateArray = isDateArray
p.deserializeXData = deserializeXData
p.deserializeYData = deserializeYData
p.getApiWhitelist = getApiWhitelist
p.getLocalWikiHost = getLocalWikiHost
p.normalizeAndValidateApiUrl = normalizeAndValidateApiUrl
p.normalizeUnsupportedExternalUrl = normalizeUnsupportedExternalUrl
p.normalizeTemplateTableSyntax = normalizeTemplateTableSyntax
p.parseWikitableCellValue = parseWikitableCellValue
p.splitCellAttrsAndContent = splitCellAttrsAndContent
p.splitInlineWikitableCells = splitInlineWikitableCells
p.parseSingleWikitable = parseSingleWikitable
p.parseWikitableFromContent = parseWikitableFromContent
p.extractRecordList = extractRecordList
p.fetchExternalData = fetchExternalData

return p