มอดูล:Convert/makeunits

-- This module generates the wikitext required at Module:Convert/data
-- by reading and processing the wikitext of the master list of units
-- (see conversion_data for the page title).
--
-- Script method:
-- * Read lines, ignoring everything before "== Conversions ==".
-- * Process the following lines:
--   * Find next level-3 heading like "=== Length ===".
--   * Parse each following line starting with "|"
--     (but ignore lines starting with "
--   * Split such lines into fields (delimiter "
") and trim
--     leading/trailing whitespace from each field.
--     Remove any "colspan" at front of second field (symbol).
--   * Remove thousand separators (commas) from the scale field.
--     If the scale is a number, do not change it.
--     Otherwise, it should be an expression like "5/9", in
--     which case it is replaced by the value of the expression.
--   * Remove wiki formatting '...' from the link field.
--   * Remove redundant fields from the unit to reduce size of data table.
--   * Create alternative forms of a unit such as an alias or a combination.
-- * Stop processing when encounter end of text or a line starting
--   with a level-2 heading ("==" but not "===").
-- * Repeat above for each heading listed at prepare_data().
-- * Output Lua source for the units table.
--
-- -- Output has the following form.
-- local all_units = {
--     "unitcode" = {                        -- standard format
--         name1    = "singular name",         -- omitted if redundant
--         name1_us = "singular name sp=us",   -- omitted if redundant
--         name2    = "plural name",           -- omitted if redundant
--         name2_us = "plural name sp=us",     -- omitted if redundant
--         symbol   = "symbol",
--         sym_us   = "symbol sp=us",          -- omitted if redundant
--         usename  = 1,                       -- omitted if empty
--         utype    = "unit type",             -- from level-3 heading
--         scale    = 1,                       -- a value, if necessary from evaluating an expression
--         subdivs  = { "ft" = { 5280, default = "km" }, "yd" = { 1760 } }  -- composite input; omitted if empty
--         link     = "title of article for wikilink",  -- omitted if empty or redundant
--         ...                                 -- other values
--     },
--     "unitcode" = {        -- alternative format to generate an alias
--         target   = "unit code",
--         ...                                 -- optional values to override those of target
--     },
--     "unitcode" = {        -- alternative format to generate a "per" unit like $/acre or BTU/h
--         per      = {u1, u2},                -- numbered table of unitcodes (u1 may be a currency symbol)
--         ...                                 -- optional values
--     },
--     "unitcode" = {        -- alternative format to generate an error message
--         shouldbe = "message that some other unit code should be used",
--     },
--     "unitcode" = {        -- alternative format for combination outputs (like 'm ft')
--         combination = {u1, u2, ...},        -- numbered table of unitcodes
--         utype    = "unit type",             -- as for standard format
--     },
--     "unitcode" = {        -- alternative format for output multiples (like 'ftin')
--         combination = {u1, u2, ...},        -- numbered table of unitcodes
--         multiple = {f1, f2, ...},           -- numbered table of integer factors
--         utype    = "unit type",             -- as for standard format
--     },
--     ...
-- }
local ulower = mw.ustring.lower
local usub = mw.ustring.sub
local text_code
local specials = {
	-- This table is used to add extra fields when defining some units which
	-- require exceptions to normal processing.
	-- Each key is in the local language, while each value is fixed text.
	-- However, this script should NOT be edited.
	-- Instead, the translation_table in Module:Convert/text can be edited,
	-- and this script will replace sections of the following with localized
	-- definitions from Module:Convert/text, if given.
	-- Ask for assistance at :en:Module talk:Convert.
	-- LATER: It would be better if this was defined in the conversion data.
	utype = {
		-- "unit type in local language" = "name_used_in_this_script"
		"fuel efficiency" = "type_fuel_efficiency",
		"length" = "type_length",
		"temperature" = "type_temperature",
		"volume" = "type_volume",
	},
	ucode = {
		exception = {
			-- "unit code in local language" = "name_used_in_module_convert"
			"ft" = "integer_more_precision",
			"in" = "subunit_more_precision",
			"lb" = "integer_more_precision",
		},
		istemperature = {
			-- Common temperature scales (not keVT or MK).
			-- "unit code in local language" = 1
			"C" = true,
			"F" = true,
			"K" = true,
			"R" = true,
		},
		usesymbol = {
			-- Use unit symbol not name if abbr not specified.
			-- "unit code in local language" = 1
			"C" = 1,
			"F" = 1,
			"K" = 1,
			"R" = 1,
			"C-change" = 1,
			"F-change" = 1,
			"K-change" = 1,
		},
		alttype = {
			-- Unit has an alternate type that is a valid conversion.
			-- "unit code in local language" = "alternate type in local language"
			"Nm" = "energy",
			"ftlb" = "torque",
			"ftlb-f" = "torque",
			"ftlbf" = "torque",
			"inlb" = "torque",
			"inlb-f" = "torque",
			"inlbf" = "torque",
			"inoz-f" = "torque",
			"inozf" = "torque",
		},
	},
}
-- Module text for the local language (localization).
-- A default table of text for enwiki is provided here.
-- If needed for another wiki, wanted sections from the table can be
-- copied into translation_table in Module:Convert/text.
-- For example, copying and modifying only the titles section may give:
--
--   local translation_table = {
--       ...             -- other items
--       mtext = {
--           titles = {
--               -- name_used_in_this_script = 'Title of page'
--               conversion_data = 'Modul:Convert/documentation/conversion data/dok',
--           },
--       },
--   }
local mtext = {
	section_names = {
		-- name_used_in_this_script = 'Section title used in conversion data'
		overrides    = 'Overrides',
		conversions  = 'Conversions',
		outmultiples = 'Output multiples',
		combinations = 'Combinations',
		inmultiples  = 'Input multiples',
		defaults     = 'Defaults',
		links        = 'Links',
		varnames     = 'Variable names',
	},
	titles = {
		-- name_used_in_this_script = 'Title of page'
		conversion_data = 'Module:Convert/documentation/conversion data/doc',
	},
	messages = {
		-- name_used_in_this_script = 'Error message ($1 = first parameter, $2 = second)'
		m_als_bad   = 'Alias has invalid text in field "$1".',
		m_als_dup   = 'Alias "$1" already defined.',
		m_als_link  = 'Alias "$1" must include a wikilink ("...") in the symlink text.',
		m_als_mul   = 'Alias "$1" has multiplier "$2" which is not a number.',
		m_als_same  = 'Should omit "$1" for alias "$2" because it is the same as its target.',
		m_als_type  = 'Target of alias "$1" has wrong type.',
		m_als_undef = 'Primary unit must be defined before alias "=$1"',
		m_cmb_miss  = 'Missing unit code for a combination.',
		m_cmb_none  = 'No units specified for combination "$1"',
		m_cmb_one   = 'Only one unit specified for combination "$1"',
		m_cmb_type  = 'Unit "$1" in combination "$2" has wrong type.',
		m_cmb_undef = 'Unit "$1" in combination "$2" not defined.',
		m_cmp_def   = 'Composite "$1" must specify a default unit code.',
		m_cmp_int   = 'Composite "$1" has components where scale ratios are not integers.',
		m_cmp_inval = 'Composite "$1" has a component with an invalid scale, "$2".',
		m_cmp_many  = 'Composite "$1" has too many fields.',
		m_cmp_miss  = 'Missing unit code for a composite.',
		m_cmp_order = 'Composite "$1" has components in wrong order or with invalid scales.',
		m_cmp_scale = 'Alternate unit "$1" in composite "$2" has wrong scale.',
		m_cmp_two   = 'Composite "$1" must specify exactly two unit codes.',
		m_cmp_type  = 'Unit "$1" in composite "$2" has wrong type.',
		m_cmp_undef = 'Unit "$1" in composite "$2" not defined.',
		m_def_cond  = 'Invalid condition in default "$1" for unit "$2".',
		m_def_fmt   = 'Default output "$1" for unit "$2" should have 2 or 3 "!".',
		m_def_rpt   = 'Default output "$1" for unit "$2" is repeated.',
		m_def_same  = 'Default output for unit "$1" is the same unit.',
		m_def_type  = 'Default output "$1" for unit "$2" has wrong type.',
		m_def_undef = 'Default output "$1" for unit "$2" is not defined.',
		m_dfs_code  = 'Defaults section: no unit code specified.',
		m_dfs_dup   = 'Defaults section: unit "$1" has already been specified.',
		m_dfs_none  = 'Defaults section: unit "$1" has no default specified.',
		m_dfs_sym   = 'Defaults section: unit "$1" must have a symbol.',
		m_dfs_two   = 'Defaults section: unit "$1" should have two fields only.',
		m_dfs_undef = 'Defaults section: unit "$1" is not defined.',
		m_dup_code  = 'Unit code "$1" has already been defined.',
		m_error     = 'Error:',
		m_ftl_read  = 'Could not read wikitext from "$1".',
		m_ftl_table = '$1 should export table "$2".',
		m_ftl_type  = 'Fatal error: unknown data type for "$1"',
		m_hdg_lev2  = 'Level 2 heading "$1" not found.',
		m_hdg_lev3  = 'No level 3 heading before: $1',
		m_line_num  = ' (line $1).',
		m_lnk_brack = 'Link "$1" has wrong number of brackets.',
		m_lnk_dup   = 'Link exception "$1" is already defined.',
		m_lnk_miss  = 'Missing unit code for a link.',
		m_lnk_none  = 'No link defined for unit "$1".',
		m_lnk_sym   = 'Unit code "$1" for a link must have a symbol.',
		m_lnk_two   = 'Row for unit "$1" link should have two fields only.',
		m_lnk_type  = 'Link exception "$1" has wrong type.',
		m_lnk_undef = 'Unit code "$1" for a link is not defined.',
		m_miss_code = 'Missing unit code.',
		m_miss_sym  = 'Missing symbol.',
		m_miss_type = 'Missing unit type.',
		m_mul_int   = 'Multiple "$1" has components where scale ratios are not integers.',
		m_mul_miss  = 'Missing unit code for a multiple.',
		m_mul_none  = 'No units specified for multiple "$1"',
		m_mul_one   = 'Only one unit specified for multiple "$1"',
		m_mul_order = 'Multiple "$1" has components in wrong order or with invalid scales.',
		m_mul_scale = 'Multiple "$1" has a component with an invalid scale, "$2".',
		m_mul_std   = 'Unit "$1" in multiple "$2" must be a standard unit.',
		m_mul_type  = 'Unit "$1" in multiple "$2" has wrong type.',
		m_mul_undef = 'Unit "$1" in multiple "$2" not defined.',
		m_no_title  = 'Need title of page with unit definitions.',
		m_ovr_dup   = 'Override "$1" is already defined.',
		m_ovr_miss  = 'Missing unit code for an override.',
		m_per_dup   = 'Per unit "$1" already defined.',
		m_per_empty = 'Unit "$1" has an empty field in the "per".',
		m_per_fuel  = 'Unit "$1" has invalid unit types for fuel efficiency.',
		m_per_two   = 'Unit "$1" does not have exactly 2 fields in the "per".',
		m_per_undef = 'Unit "$1" has undefined unit code "$2" in the "per".',
		m_percent_s = 'Field "$1" must not contain "%s".',
		m_pfx_bad   = 'Unknown prefix: "$1".',
		m_pfx_name  = 'Unit with Prefix set must include Name.',
		m_scl_bad   = 'Scale expression is invalid: "$1".',
		m_scl_miss  = 'Missing scale.',
		m_scl_oflow = 'Scale expression gives an invalid value: "$1".',
		m_var_cnt   = 'Variable names section: each row must have five fields.',
		m_var_dup   = 'Unit "$1" already has a variable name.',
		m_var_miss  = 'Missing field for a variable name.',
		m_var_undef = 'Unit "$1" in variable names is not defined.',
		m_warning   = 'Warning:',
		m_wrn_more  = '  (and more not shown)',
		m_wrn_nbsp  = 'Line $1 contains a nonbreaking space.',
		m_wrn_nodef = 'Units with the following unit codes have no default output.',
		m_wrn_ucode = '  $1',
	},
}
local function message(key, ...)
	-- Return a message from the message table, which can be localized.
	-- '$1', '$2', ... are replaced with the first, second, ... parameters,
	-- each of which must be a string or a number.
	-- The global variable is_test_run can be set by a testing program to
	-- check the messages generated by this program.
	local rep = {}
	for i, v in ipairs({...}) do
		rep'$' .. i = v
	end
	key = key or '???'
	local extra
	if is_test_run and key ~= 'm_line_num' then
		extra = key .. ': '
	else
		extra = 
	end
	return extra .. string.gsub(mtext.messageskey or key, '$%d+', rep)
end
local function quit(key, ...)
	-- Use error() to pass an error message to the surrounding pcall().
	error(message(key, ...), 0)
end
local function quit_no_message()
	-- Throw an error.
	-- This is used in some functions which can throw an error with a message,
	-- but where the message is in fact never displayed because the calling
	-- function uses pcall to catch errors, and any message is ignored.
	-- Using this function documents that the message (which may be useful in
	-- some other application) does not need translation as it never appears.
	error('this message is not displayed', 0)
end
local function collection()
	-- Return a table to hold items.
	return {
		n = 0,
		add = function (self, item)
			self.n = self.n + 1
			selfself.n = item
		end,
		pop = function (self, item)
			if self.n > 0 then
				local top = selfself.n
				self.n = self.n - 1
				return top
			end
		end,
		join = function (self, sep)
			return table.concat(self, sep or '\n')
		end,
	}
end
local warnings = collection()
local function add_warning(key, ...)
	-- Add a warning that will be inserted before the final result.
	warnings:add(message(key, ...))
end
---Begin code to evaluate expressions-----------------------------------
-- This is needed because Lua's loadstring() is not available in Scribunto,
-- and each scale value can be specifed as an expression such as "5/9".
-- More complex expressions are supported, including use of parentheses
-- and the binary operators: + - * / ^
local operators = {
	'+' = { precedence = 1, associativity = 1, func = function (a, b) return a + b end },
	'-' = { precedence = 1, associativity = 1, func = function (a, b) return a - b end },
	'*' = { precedence = 2, associativity = 1, func = function (a, b) return a * b end },
	'/' = { precedence = 2, associativity = 1, func = function (a, b) return a / b end },
	'^' = { precedence = 3, associativity = 2, func = function (a, b) return a ^ b end },
	'(' = '(',
	')' = ')',
}
local function tokenizer(text)
	-- Function 'next' returns the next token which is one of:
	--     number
	--     table (operator)
	--     string ('(' or ')')
	--     nil (end of text)
	-- If invalid, an error is thrown.
	-- The number is unsigned (unary operators are not supported).
	return {
		pos = 1,
		maxpos = #text,
		text = text,
		next = function(self)
			if self.pos  0 and (opstackopstack.n.precedence > precedence or
			(opstackopstack.n.precedence == precedence and associativity == 1)) do
			local rhs = numstack:pop()
			local lhs = numstack:pop()
			if not (rhs and lhs) then quit_no_message('missing number') end
			local op = opstack:pop()
			numstack:add(op.func(lhs, rhs))
		end
	end
	local token_last
	local function set_state(token_type)
		if token_last == token_type then
			local missing = (token_type == 'number') and 'operator' or 'number'
			quit_no_message('missing ' .. missing)
		end
		token_last = token_type
	end
	while true do
		local token = tokens:next()
		if type(token) == 'number' then
			set_state('number')
			numstack:add(token)
		elseif type(token) == 'table' then
			set_state('operator')
			perform_ops(token.precedence, token.associativity)
			opstack:add(token)
		elseif token == '(' then
			set_state('number')
			numstack:add(evaluate_tokens(tokens, true))
		elseif token == ')' then
			if inparens then
				break
			end
			quit_no_message('unbalanced parentheses')
		else
			break
		end
	end
	perform_ops(0)
	if numstack.n > 1 then quit_no_message('missing operator') end
	if numstack.n ' or '>='
	-- In addition, the following form is supported:
	--    LHS and RHS
	-- where
	--    LHS, RHS = any of above expressions.
	local function compare(value, text)
		local arithop, factor, compop, limit = text:match('^%s*v%s*(*?)(.-)(=?)(.*)$')
		if arithop == nil then
			quit_no_message('Invalid default expression.')
		elseif arithop == '*' then
			factor = tonumber(factor)
			if factor == nil then
				quit_no_message('Invalid default expression.')
			end
			value = value * factor
		end
		limit = tonumber(limit)
		if limit == nil then
			quit_no_message('Invalid default expression.')
		end
		if compop ==  then
			return value > limit
		elseif compop == '>=' then
			return value >= limit
		end
		quit_no_message('Invalid default expression.')  -- should not occur
	end
	local lhs, rhs = condition:match('^(.-%W)and(%W.*)')
	if lhs == nil then
		return compare(value, condition)
	end
	return compare(value, lhs) and compare(value, rhs)
end
---End adapted code-----------------------------------------------------
local function strip(text)
	-- Return text with no leading/trailing whitespace.
	return text:match("^%s*(.-)%s*$")
end
local function empty(text)
	-- Return true if text is nil or empty (assuming a string).
	return text == nil or text == 
end
-- Tables of units: k = unit code, v = unit table.
local units_index = {}  -- all units: normal, alias, per, combination, or multiple
local alias_index = {}  -- all aliases (to detect attempts to define more than once)
local per_index = {}    -- all "per" units (to detect attempts to define more than once)
local function get_unit(ucode)
	-- Look up unit code in our cache of units.
	if empty(ucode) then
		return nil
	end
	return lookup(units_index, ucode)
end
local overrides = {}  -- read from input for unit codes that should not be checked for a duplicate
local function insert_unique_unit(data, unit, index)
	-- After inserting any required built-in data, insert the unit into the
	-- data table and (if index not nil) add to index,
	-- but not if the unit code is already defined.
	local ucode = unit.unitcode
	local known = get_unit(ucode)
	if known and not overridesucode then
		quit('m_dup_code', ucode)
	end
	for item, t in pairs(specials.ucode) do
		unititem = tucode
	end
	if index then
		indexucode = unit
	end
	table.insert(data, unit)
end
local function check_condition(condition)
	-- Return true if condition appears to be valid; otherwise return false.
	for _, value in ipairs({ 0, 0.1, 1, 1.1, 10, 100, 1000, 1e4, 1e5 }) do
		local success, result = pcall(evaluate_condition, value, condition)
		if not success then
			return false
		end
	end
	return true
end
local function check_default_expression(default, ucode)
	-- Return a numbered table of names present in param default
	-- (two names if an expression, or one name (param default) otherwise).
	-- Throw an error if a problem occurs.
	-- An expression uses pipe-delimited fields with 'v' representing
	-- the input value for the conversion.
	-- Example (suffix is optional): 'v = maxerrors then
						break
					end
				end
			end
		end
	end
	if errors.n > 0 then
		error(errors:join(), 0)
	end
	if missing.n > 0 then
		add_warning('m_wrn_nodef')
		local limit = maxerrors
		for _, v in ipairs(missing) do
			limit = limit - 1
			if limit = maxerrors then
			break
		end
	end
	if errors.n > 0 then
		error(errors:join(), 0)
	end
end
local function update_units(units, composites, varnames)
	-- Update some unit definitions with extra data defined in other sections.
	-- This is done after all input data has been processed.
	for _, unit in ipairs(units) do
		local comp = compositesunit.unitcode
		if comp then
			unit.subdivs = '{ ' .. table.concat(comp.subdivs, ', ') .. ' }'
		end
		local vn = varnamesunit.unitcode
		if vn then
			unit.varname = vn
		end
	end
end
local function make_override(data)
	-- Return a function which, when called, stores a unit code that is not to be
	-- checked for a duplicate. The table is stored in data (also a table).
	return function (utype, fields)
		local ucode = fields1
		if empty(ucode) then
			quit('m_ovr_miss')
		end
		if dataucode then
			quit('m_ovr_dup', ucode)
		end
		dataucode = true
	end
end
local function make_default(data)
	-- Return a function which, when called, stores a table that defines a
	-- default output unit. The table is stored in data (also a table).
	local defaults_index = {}  -- to detect attempts to define a default twice
	return function (utype, fields)
		-- Store a table defining a unit.
		-- This is for a unit such as 'kg' that has a default output unit
		-- different from what is defined for the base unit ('g').
		-- Throw an error if a problem occurs.
		local ucode = fields1
		local default = fields2
		if empty(ucode) then
			quit('m_dfs_code')
		end
		if empty(default) then
			quit('m_dfs_none', ucode)
		end
		if #fields ~= 2 then
			quit('m_dfs_two', ucode)
		end
		local unit_table = get_unit(ucode)
		if not unit_table then
			quit('m_dfs_undef', ucode)
		end
		local symbol = unit_table.defkey or unit_table.symbol
		if empty(symbol) then
			quit('m_dfs_sym', ucode)
		end
		check_default(default, ucode, utype, unit_table)
		if defaults_indexucode then
			quit('m_dfs_dup', ucode)
		end
		defaults_indexucode = default
		table.insert(data, { symbol = symbol, default = default })
	end
end
local function clean_link(link, name)
	-- Return link, customary where:
	--   link = given link after removing any '...' wiki formatting
	--          and removing any leading '+' or '*' or '@';
	--   customary = 1 if leading '+', or 2 if '*' or 3 if '@', or nil
	--   (for extra "US" or "U.S." or "Imperial" customary units link).
	-- Result has leading/trailing whitespace removed, and is nil if empty
	-- or if link matches the name, if a name is specified.
	-- Exception: If the link is empty and the name starts with ,
	-- the link is stored as  (for a unit name which is always linked).
	-- If the resulting link is nil, no link field is stored, and
	-- if a link is required, it will be set from the unit's name.
	local original = link
	if empty(link) then
		return (name and name:sub(1, 2) == ) and  or nil
	end
	local prefixes = { '+' = 1, '*' = 2, '@' = 3 }
	local customary = prefixeslink:sub(1, 1)
	if customary then
		link = strip(link:sub(2))
	end
	if link:sub(1, 2) ==  then
		link = link:sub(3)
	end
	if link:sub(-2) ==  then
		link = link:sub(1, -3)
	end
	link = strip(link)
	if link:sub(1, 1) ==  or link:sub(-1) ==  then
		quit('m_lnk_brack', original)
	end
	if link ==  then
		link = nil
	elseif name then
		local l = ulower(usub(link, 1, 1)) .. usub(link, 2)
		local n = ulower(usub(name, 1, 1)) .. usub(name, 2)
		if l == n then
			link = nil  -- link == name, ignoring case of first letter
		end
	end
	return link, customary
end
local function make_link(data)
	-- Return a function which, when called, stores a table that defines a
	-- link exception. The table is stored in data (also a table).
	local links_index = {}  -- to detect attempts to define a link twice
	return function (utype, fields)
		-- Store a table defining a unit.
		-- This is for a unit such as 'kg' that has a linked article
		-- different from what is defined for the base unit ('g').
		-- Throw an error if a problem occurs.
		local ucode = fields1
		local link = clean_link(fields2)
		if empty(ucode) then
			quit('m_lnk_miss')
		end
		if empty(link) then
			quit('m_lnk_none', ucode)
		end
		if #fields ~= 2 then
			quit('m_lnk_two', ucode)
		end
		local unit_table = get_unit(ucode)
		if not unit_table then
			quit('m_lnk_undef', ucode)
		end
		if utype ~= unit_table.utype then
			quit('m_lnk_type', ucode)
		end
		local symbol = unit_table.symbol
		if empty(symbol) then
			quit('m_lnk_sym', ucode)
		end
		if links_indexucode then
			quit('m_lnk_dup', ucode)
		end
		links_indexucode = link
		table.insert(data, { symbol = symbol, link = link })
	end
end
local function clean_scale(scale)
	-- Return cleaned scale as a string, after evaluating any expression.
	-- It would be better to retain scale expressions like "5/9" so that
	-- the expression is evaluated on the server and maintains the full
	-- resolution of the server. However, there are many such expressions
	-- in the table of all units, and it seems pointless to require the
	-- server to evaluate all of them just to do one convert.
	if empty(scale) then
		quit('m_scl_miss')
	end
	assert(type(scale) == 'string', 'Bug: scale has an unexpected type')
	scale = string.gsub(scale, ',', )  -- remove comma separators
	if tonumber(scale) then  -- not an expression
		return scale
	end
	local status, value = pcall(evaluate, scale)
	if not (status and type(value) == 'number') then
		quit('m_scl_bad', scale)
	end
	local result = string.format('%.17g', value)
	if result:find('#n') then
		-- Lua can give results like "#INF" while Scribunto gives "inf". Either is an error.
		quit('m_scl_oflow', scale)
	end
	-- Omit redundant zeros from results like '1.2e-005'.
	-- Do not bother looking for results like '1.2e+005' as none occur in practice.
	local lhs, zeros, rhs = result:match('^(.-e%-)(0+)(.*)')
	if zeros then
		result = lhs .. rhs
	end
	return result
end
local function add_alias_optional_fields(unit, start, fields, target)
	-- Inspect fieldsi for i = start, start+1 ..., and extract any
	-- definitions appropriate for an alias or "per", and add them to unit.
	-- For an alias, target is a valid unit; for a "per", target is nil.
	-- Throw error if encounter an invalid entry.
	for i = start, #fields do
		local field = fieldsi
		if not empty(field) then
			local lhs, rhs = field:match('^%s*(.-)%s*=%s*(.-)%s*$')
			local good
			if not empty(rhs) then
				for _, item in ipairs({ 'sp', 'default', 'link', 'multiplier', 'symbol', 'symlink' }) do
					if lhs == item then
						if item == 'sp' then
							if rhs == 'us' then
								unit.sp_us = true
								good = true
							end
						elseif item == 'link' then
							local tlink
							if target then
								tlink = targetitem
							end
							local link, customary = clean_link(rhs, tlink)
							if link then
								unititem = link
							end
							if customary then
								unit.customary = customary
							end
							good = true
						elseif item == 'symlink' then
							local pos1 = rhs:find(, 1, true)
							local pos2 = rhs:find(, 1, true)
							if not (pos1 and pos2 and (pos1  1e-6 then
				quit('m_cmp_int', unitcode)
			end
			ratiosi = rounded
		end
		local text = { tostring(ratios1) }
		local function add_text(key, value)
			table.insert(text, string.format('%s = %q', key, value))
		end
		if default_code then
			add_text('default', default_code)
		end
		if alternate_code then
			add_text('unit', alternate_code)
		end
		if fixed_name then
			add_text('name', fixed_name)
		end
		local subdiv = string.format('"%s" = { %s }', ucodes2, table.concat(text, ', '))
		local main_code = ucodes1
		local item = datamain_code
		if item then
			table.insert(item.subdivs, subdiv)
		else
			datamain_code = { subdivs = { subdiv } }
		end
	end
end
local function make_outputmultiple(data)
	-- Return a function which, when called, stores a table that defines a
	-- single multiple output unit. The table is stored in data (also a table).
	return function (utype, fields)
		-- Store a table defining a unit.
		-- This is for a multiple unit like 'ydftin' (result in yards, feet, inches).
		-- The target units must be defined first.
		-- Throw an error if a problem occurs.
		local unit = { utype = utype }
		local ucodes, scales = {}, {}
		for i, v in ipairs(fields) do
			if i == 1 then  -- unitcode
				if v ==  then
					quit('m_mul_miss')
				end
				unit.unitcode = v
			elseif v ==  then
				-- Ignore empty fields.
			else
				local target = get_unit(v)
				if not target then
					quit('m_mul_undef', v, unit.unitcode)
				end
				if target.utype ~= utype then
					quit('m_mul_type', v, unit.unitcode)
				end
				if not target.scale then
					quit('m_mul_std', v, unit.unitcode)
				end
				table.insert(ucodes, v)
				table.insert(scales, target.scale)
			end
		end
		if #ucodes  1e-6 then
				quit('m_mul_int', unit.unitcode)
			end
			ratiosi = rounded
		end
		unit.combination = reversed(ucodes)
		unit.multiple = reversed(ratios)
		insert_unique_unit(data, unit, units_index)
	end
end
-- To make updating the data module easier, this script inserts a preamble
-- and a postamble so the result can be used to replace the whole page.
local data_preamble = =
-- Conversion data used by Module:Convert which uses mw.loadData() for
-- read-only access to this module so that it is loaded only once per page.
-- See :en:Template:Convert/Transwiki guide if copying to another wiki.
--
-- These data tables follow:
--   all_units           all properties for a unit, including default output
--   default_exceptions  exceptions for default output ('kg' and 'g' have different defaults)
--   link_exceptions     exceptions for links ('kg' and 'g' have different links)
--
-- These tables are generated by a script which reads the wikitext of a page that
-- documents the required properties of each unit; see :en:Module:Convert/doc.
**
local data_postamble = =
return {
	all_units = all_units,
	default_exceptions = default_exceptions,
	link_exceptions = link_exceptions,
}=
local out_unit_prefix = 
---------------------------------------------------------------------------
-- Do not change the data in this table because it is created by running --
-- a script that reads the wikitext from a wiki page (see note above).   --
---------------------------------------------------------------------------
local all_units = {
local out_unit_suffix = 
}
local out_default_prefix = 
---------------------------------------------------------------------------
-- Do not change the data in this table because it is created by running --
-- a script that reads the wikitext from a wiki page (see note above).   --
---------------------------------------------------------------------------
local default_exceptions = {
	-- Prefixed units with a default different from that of the base unit.
	-- Each key item is a prefixed symbol (unitcode for engineering notation).
local out_default_suffix = 
}
local out_default_item = 
	"{symbol}" = "{default}",
local out_link_prefix = 
---------------------------------------------------------------------------
-- Do not change the data in this table because it is created by running --
-- a script that reads the wikitext from a wiki page (see note above).   --
---------------------------------------------------------------------------
local link_exceptions = {
	-- Prefixed units with a linked article different from that of the base unit.
	-- Each key item is a prefixed symbol (not unitcode).
local out_link_suffix = 
}
local out_link_item = 
	"{symbol}" = "{link}",
local combination_specification = {     -- pure combination like 'm ft', or a multiple like 'ftin'
	'combination',
	'multiple',
	'utype',
}
local alias_specification = {
	'target',
	'symbol',
	'sp_us',
	'default',
	'link',
	'symlink',
	'customary',
	'multiplier',
}
local per_specification = {
	'per',
	'symbol',
	'sp_us',
	'utype',
	'invert',
	'iscomplex',
	'default',
	'link',
	'symlink',
	'customary',
	'multiplier',
}
local shouldbe_specification = {
	'shouldbe',
}
local unit_specification = {
	'_name1',
	'_name1_us',
	'_name2',
	'_name2_us',
	'_symbol',
	'_sym_us',
	'prefix_position',
	'name1',
	'name1_us',
	'name2',
	'name2_us',
	'varname',
	'symbol',
	'sym_us',
	'usename',
	'usesymbol',
	'utype',
	'alttype',
	'builtin',
	'scale',
	'offset',
	'invert',
	'iscomplex',
	'istemperature',
	'exception',
	'prefixes',
	'default',
	'subdivs',
	'defkey',
	'linkey',
	'link',
	'customary',
	'sp_us',
}
local no_quotes = {
	combination = true,
	customary = true,
	multiple = true,
	multiplier = true,
	offset = true,
	per = true,
	prefix_position = true,
	scale = true,
	subdivs = true,
}
local function add_unit_lines(results, unit, spec)
	-- Add lines of Lua source to define a unit to the results collection.
	local function add_line(line)
		-- Had planned to replace sequences of spaces with 4-column tabs here
		-- (because the CodeEditor now assumes the use of such tabs).
		-- However, 4-column tabs are only visible when editing a module
		-- with browser scripting and the CodeEditor enabled, and that is rare.
		-- A module is usually viewed (with 8-column tabs), and some indents
		-- would be messed up unless 8-column tabs are used. Therefore,
		-- have decided to simply replace 8 spaces at start of line with a single
		-- tab which reduces the size of the module, and is correct for viewing.
		if line:sub(1, 8) == string.rep(' ', 8) then
			line = '\t' .. line:sub(9)
		end
		results:add(line)
	end
	local first_item = '    "' .. unit.unitcode .. '" = {'
	local last_item  = '    },'
	add_line(first_item)
	for _, k in ipairs(spec) do
		local v = unitk
		if v then
			local want_quotes = (type(v) == 'string' and not no_quotesk)
			if type(v) == 'boolean' then
				v = tostring(v)
			elseif type(v) == 'number' or k == 'scale' then
				-- Replace results like '1e-006' with '1e-6'.
				v = string.gsub(tostring(v), '(e+-)0+(1-9.*)', '%1%2', 1)
			elseif type(v) ~= 'string' then
				quit('m_ftl_type', unit.unitcode)
			end
			local fmt = string.format('%8s%%-9s= %%%s,', , want_quotes and 'q' or 's')
			add_line(fmt:format(k, v))
		end
	end
	add_line(last_item)
end
local function numbered_table_as_string(data, unit)
	local t = {}
	for _, v in ipairs(data) do
		if type(v) == 'string' then
			table.insert(t, '"' .. v .. '"')
		elseif type(v) == 'number' then
			table.insert(t, tostring(v))
		else
			quit('m_ftl_type', unit.unitcode)
		end
	end
	return '{ ' .. table.concat(t, ', ') .. ' }'
end
local function extract_heading(line)
	-- Return n, s where n = heading level number (nil if none), and
	-- s = heading text (with leading/trailing whitespace removed).
	local pattern = '^(==+)%s*(.-)%s*(==+)%s*$'
	local before, heading, after = line:match(pattern)
	if heading and #heading > 0 then
		-- Don't bother checking if before == after.
		return #before, heading
	end
end
local function fields(line)
	-- Return a numbered table of fields split from line.
	-- Items are delimited by "
".
	-- Each item has leading/trailing whitespace removed, and any encoded pipe
	-- characters are decoded.
	-- The second field (for symbol when processing units) is adjusted to
	-- remove any "colspan" at the front of lines like:
	-- "| unitcode 

Text to display for an error message".
	local t = {}
	line = line .. "
"  -- to get last field
	for item in line:gmatch("%s*(.-)%s
") do
		table.insert(t, (item:gsub('|', '|')))
	end
	if t2 then
		local cleaned = t2:match('^%s*colspan%s*=.-|%s*(.*)$')
		if cleaned then
			t2 = cleaned
		end
	end
	return t
end
local function prepare_section(maker, lines, section, maxerrors, need_section, need_utype)
	-- Process the first level-two section with the given section name
	-- in the given table of lines of wikitext.
	-- If successful, maker inserts each item into a table.
	-- Otherwise, an error is thrown.
	local skip = true
	local errors = collection()
	local utype  -- unit type (from level-three heading)
	local nbsp = '\194\160'  -- nonbreaking space is utf-8 encoded as hex c2 a0
	for linenumber, line in ipairs(lines) do
		if skip then
			-- Skip down to and including the starting heading.
			local level, heading = extract_heading(line)
			if level == 2 and heading == section then
				skip = false
			end
		else
			-- Accummulate unit definitions.
			local c1 = line:sub(1, 1)
			local c2 = line:sub(2, 2)
			if c1 == '|' and not (c2 == '-' or c2 == '}') then
				if need_utype and empty(utype) then
					quit('m_hdg_lev3', line)
				end
				if line:find(nbsp, 1, true) then
					-- For example, "acre ft" does not work if it contains nbsp.
					add_warning('m_wrn_nbsp', linenumber)
				end
				local ok, msg = pcall(maker, utype, fields(line:sub(2)))
				if not ok then
					if msg:sub(-1) == '.' then msg = msg:sub(1, -2) end
					errors:add(msg .. message('m_line_num', linenumber))
					if errors.n >= maxerrors then
						break
					end
				end
			else
				local level, heading = extract_heading(line)
				if level == 3 then
					utype = ulower(heading)
				elseif level == 2 then
					break
				end
			end
		end
	end
	if skip and need_section then
		quit('m_hdg_lev2', section)
	end
	if errors.n > 0 then
		error(errors:join(), 0)
	end
end
local function get_page_lines(page_title)
	-- Read the wikitext of the page at the given title; split the text into
	-- lines with leading and trailing space removed from each line.
	-- Return a numbered table of the lines, or throw an error.
	if empty(page_title) then
		quit('m_no_title')
	end
	local t = mw.title.new(page_title)
	if t then
		local content = t:getContent()
		if content then
			if content:sub(-1) ~= '\n' then
				content = content .. '\n'
			end
			local lines = collection()
			for line in string.gmatch(content, '\t *(.-)\t\r *\n') do
				lines:add(line)
			end
			return lines
		end
	end
	quit('m_ftl_read', page_title)
end
local function prepare_data(conversion_data_title, maxerrors, is_sandbox)
	-- Read the page of conversion data, and process the wikitext
	-- in the sections with wanted level-two headings.
	-- Return units, defaults, links (three tables).
	-- Throw an error if a problem occurs.
	local composites, defaults, links, units, varnames = {}, {}, {}, {}, {}
	local sections = {
		{ 'overrides'   , make_override      , overrides , 0 },
		{ 'conversions' , make_unit          , units     , 0 },
		{ 'outmultiples', make_outputmultiple, units     , 0 },
		{ 'combinations', make_combination   , units     , 0 },
		{ 'inmultiples' , make_inputmultiple , composites, 0 },  -- after all units defined so default will be defined
		{ 'defaults'    , make_default       , defaults  , 0 },
		{ 'links'       , make_link          , links     , 0 },
		{ 'varnames'    , make_varname       , varnames  , 1 },
	}
	local lines = get_page_lines(conversion_data_title)
	for _, section in ipairs(sections) do
		local heading = mtext.section_namessection1
		local maker = section2(section3)
		local code = section4
		local need_section, need_utype
		if code == 0 and not is_sandbox then
			need_section = true
		end
		if code == 0 then
			need_utype = true
		end
		prepare_section(maker, lines, heading, maxerrors, need_section, need_utype)
	end
	check_all_defaults(units, maxerrors)
	check_all_pers(units, maxerrors)
	update_units(units, composites, varnames)
	return units, defaults, links
end
local function _makeunits(results, data_title, text_title)
	-- Read the wikitext for the conversion data.
	-- Append output to given results collection, or throw error if a problem.
	text_code = require(text_title)
	for _, name in ipairs({ 'SIprefixes', 'eng_scales', 'currency' }) do
		if type(text_codename) ~= 'table' then
			quit('m_ftl_table', text_title, name)
		end
	end
	local translation = text_code.translation_table
	if translation then
		if translation.plural_suffix then
			plural_suffix = translation.plural_suffix
		end
		local ts = translation.specials
		if ts then
			if ts.utype then
				specials.utype = ts.utype
			end
			if ts.ucode then
				specials.ucode = ts.ucode
			end
		end
		local tm = translation.mtext
		if tm then
			if tm.section_names then
				mtext.section_names = tm.section_names
			end
			if tm.titles then
				mtext.titles = tm.titles
			end
			if tm.messages then
				mtext.messages = tm.messages
			end
		end
	end
	local is_sandbox
	local conversion_data_title = mtext.titles.conversion_data
	if data_title and conversion_data_title ~= data_title then
		conversion_data_title = data_title
		if is_test_run then
			is_sandbox = true
			data_preamble = nil
			data_postamble = nil
			out_unit_prefix = 'local all_units = {'
			out_unit_suffix = '}'
			out_default_prefix = '\nlocal default_exceptions = {'
			out_default_suffix = '}'
			out_default_item = '\t"{symbol}" = "{default}",'
			out_link_prefix = '\nlocal link_exceptions = {'
			out_link_suffix = '}'
			out_link_item = '\t"{symbol}" = "{link}",'
		end
	end
	local units, defaults, links = prepare_data(conversion_data_title, 20, is_sandbox)
	if data_preamble then
		results:add(data_preamble)
	end
	results:add(out_unit_prefix)
	for _, unit in ipairs(units) do
		local spec
		if unit.target then
			spec = alias_specification
		elseif unit.per then
			spec = per_specification
			unit.per = numbered_table_as_string(unit.per, unit)
		elseif unit.shouldbe then
			spec = shouldbe_specification
		elseif unit.combination then
			spec = combination_specification
			unit.combination = numbered_table_as_string(unit.combination, unit)
			if unit.multiple then
				unit.multiple = numbered_table_as_string(unit.multiple, unit)
			end
		else
			spec = unit_specification
		end
		add_unit_lines(results, unit, spec)
	end
	results:add(out_unit_suffix)
	if #defaults > 0 or not is_sandbox then
		results:add(out_default_prefix)
		for _, unit in ipairs(defaults) do
			results:add((out_default_item:gsub('{(%w_+)}', unit)))
		end
		results:add(out_default_suffix)
	end
	if #links > 0 or not is_sandbox then
		results:add(out_link_prefix)
		for _, unit in ipairs(links) do
			results:add((out_link_item:gsub('{(%w_+)}', unit)))
		end
		results:add(out_link_suffix)
	end
	if data_postamble then
		results:add(data_postamble)
	end
end
local function makeunits(frame)
	local args = frame.args
	local results = collection()
	local ok, msg = pcall(_makeunits, results, args1, args2 or 'Module:Convert/text')
	if not ok then
		results:add(message('m_error'))
		results:add()
		results:add(msg)
	end
	local warn = 
	if warnings.n > 0 then
		warn = message('m_warning') .. '\n\n' .. warnings:join() .. '\n\n'
	end
	-- Pre tags returned by a module are html tags, not like wikitext ....
	-- The following renders the text as is, and preserves tab characters.
	return '\n' .. mw.text.nowiki(warn .. results:join()) .. '\n\n'
end
return { makeunits = makeunits }