-- @param section Required. Title of the section to get (in wikitext), for example 'History' or 'History of Athens'.
-- @return Wikitext of the requested section.
local function getSection(text, section)
section = mw.text.trim(section)
local escapedSection = escapeString(section)
-- First check if the section title matches a <section> tag
if string.find(text, '<%s*[Ss]ection%s+begin%s*=%s*["\']?%s*' .. escapedSection .. '%s*["\']?%s*/>') then -- avoid expensive search if possible
text = mw.text.trim((text
:gsub('<%s*[Ss]ection%s+end=%s*["\']?%s*'.. escapedSection ..'%s*["\']?%s*/>.-<%s*[Ss]ection%s+begin%s*=%s*["\']?%s*' .. escapedSection .. '%s*["\']?%s*/>', '') -- remove text between section tags
:gsub('^.-<%s*[Ss]ection%s+begin%s*=%s*["\']?%s*' .. escapedSection .. '%s*["\']?%s*/>', '') -- remove text before first section tag
:gsub('<%s*[Ss]ection%s+end=%s*["\']?%s*'.. escapedSection ..'%s*["\']?%s*/>.*', '') -- remove text after last section tag
))
if text == '' then return throwError('section-tag-empty', section) end
return text
end
local level, text = string.match('\n' .. text .. '\n', '\n(==+)%s*' .. escapedSection .. '%s*==.-\n(.*)')
if not text then return throwError('section-not-found', section) end
local nextSection = '\n==' .. string.rep('=?', #level - 2) .. '[^=].*'
text = string.gsub(text, nextSection, '') -- remove later sections with headings at this level or higher
text = mw.text.trim(text)
if text == '' then return throwError('section-empty', section) end
return text
end  
-- Replace the first call to each reference defined outside of the text for the full reference, to prevent undefined references
-- Then prefix the page title to the reference names to prevent conflicts
-- that is, replace <ref name="Foo"> for <ref name="Title of the article Foo">
-- and also for
-- also remove reference groups: <ref name="Foo" group="Bar"> for <ref name="Title of the article Foo">
-- and <ref group="Bar"> for <ref>
-- @todo The current regex may fail in cases with both kinds of quotes, like <ref name="Darwin's book">
local function fixReferences(text, page, full)
if not full then full = getText(page) end
local refNames = {}
local refName
local refBody
local position = 1
while position < mw.ustring.len(text) do
refName, position = mw.ustring.match(text, '<%s*[Rr][Ee][Ff][^>]*name%s*=%s*["\']?([^"\'>]+)["\']?[^>]*/%s*>', position)
if refName then
refName = mw.text.trim(refName)
if not refNames[refName] then -- make sure we process each ref name only once
table.insert(refNames, refName)
refName = escapeString(refName)
refBody = mw.ustring.match(text, '<%s*[Rr][Ee][Ff][^>]*name%s*=%s*["\']?%s*' .. refName .. '%s*["\']?[^>/]*>.-<%s*/%s*[Rr][Ee][Ff]%s*>')
if not refBody then -- the ref body is not in the excerpt
refBody = mw.ustring.match(full, '<%s*[Rr][Ee][Ff][^>]*name%s*=%s*["\']?%s*' .. refName .. '%s*["\']?[^/>]*>.-<%s*/%s*[Rr][Ee][Ff]%s*>')
if refBody then -- the ref body was found elsewhere
text = mw.ustring.gsub(text, '<%s*[Rr][Ee][Ff][^>]*name%s*=%s*["\']?%s*' .. refName .. '%s*["\']?[^>]*/?%s*>', mw.ustring.gsub(refBody, '%%', '%%%%'), 1)
end
end
end
else
position = mw.ustring.len(text)
end
end
page = string.gsub(page, '"', '') -- remove any quotation marks from the page title
text = mw.ustring.gsub(text, '<%s*[Rr][Ee][Ff][^>]*name%s*=%s*["\']?([^"\'>/]+)["\']?[^>/]*(/?)%s*>', '<ref name="' .. page .. ' %1"%2>')
text = mw.ustring.gsub(text, '<%s*[Rr][Ee][Ff]%s*group%s*=%s*["\']?[^"\'>/]+["\']%s*>', '<ref>')
return text
end  
-- Replace the bold title or synonym near the start of the page by a link to the page
local function linkBold(text, page)
local lang = mw.language.getContentLanguage
local position = mw.ustring.find(text, "**" .. lang:ucfirst(page) .. "**", 1, true) -- look for "**Foo** is..." (uc) or "A **foo** is..." (lc)
or mw.ustring.find(text, "**" .. lang:lcfirst(page) .. "**", 1, true) -- plain search: special characters in page represent themselves
if position then
local length = mw.ustring.len(page)
text = mw.ustring.sub(text, 1, position + 2) .. "" .. mw.ustring.sub(text, position + 3, position + length + 2) .. "" .. mw.ustring.sub(text, position + length + 3, -1) -- link it
else -- look for anything unlinked in bold, assumed to be a synonym of the title (e.g. a person's birth name)
text = mw.ustring.gsub(text, "**(.-'*)**", function(a, b)
if not mw.ustring.find(b, "%[") and not mw.ustring.find(b, "%{") then -- if not wikilinked or some weird template
return "**" .. b .. "**" -- replace **Foo** by **Foo**
else
return nil -- instruct gsub to make no change
end
end, 1) -- "end" here terminates the anonymous replacement function(a, b) passed to gsub
end
return text
end  
-- Remove non-free files.
-- @param text Required. Wikitext to clean.
-- @return Clean wikitext.
local function removeNonFreeFiles(text)
local fileNamespaces = getNamespaces('File')
local fileName
local fileDescription
local = mw.getCurrentFrame
for file in string.gmatch(text, '%b[]') do
if matchAnyLink(file, fileNamespaces) then
fileName = 'File:' .. string.match(file, '%[%[[^:]-:([^]|]+)')
fileDescription, fileName = getText(fileName)
if fileName then
if not fileDescription or fileDescription == '' then
fileDescription = :preprocess('') -- try Commons
end
if fileDescription and string.match(fileDescription, '[Nn]on%-free') then
text = removeString(text, file)
end
end
end
end
return text
end  
-- Remove any self links
local function removeSelfLinks(text)
local lang = mw.language.getContentLanguage
local page = escapeString(mw.title.getCurrentTitle.prefixedText)
local ucpage = lang:ucfirst(page)
local lcpage = lang:lcfirst(page)
text = text
:gsub('%[%[(' .. ucpage .. ')%]%]', '%1')
:gsub('%[%[(' .. lcpage .. ')%]%]', '%1')
:gsub('%[%[' .. ucpage .. '|([^]]+)%]%]', '%1')
:gsub('%[%[' .. lcpage .. '|([^]]+)%]%]', '%1')
return text
end  
-- Remove all wikilinks
local function removeLinks(text)
text = text
:gsub('%[%[[^%]|]+|([^]]+)%]%]', '%1')
:gsub('%[%[([^]]+)%]%]', '%1')
:gsub('%[[^ ]+ ([^]]+)%]', '%1')
:gsub('%[([^]]+)%]', '%1')
return text
end  
-- Remove HTML comments
local function removeComments(text)
text = string.gsub(text, '<!%-%-.-%-%->', '')
return text
end  
-- Remove behavior switches, such as __NOTOC__
local function removeBehaviorSwitches(text)
text = string.gsub(text, '__[A-Z]+__', '')
return text
end  
-- Remove bold text
local function removeBold(text)
text = string.gsub(text, "*'", *)
return text
end  
-- Main function for modules
local function get(page, options)
if not options then options = {} end  
-- Make sure the page exists
if not page then return throwError('no-page') end
page = mw.text.trim(page)
page = mw.text.decode(page)
if page == '' then return throwError('no-page') end
local page, hash, section = string.match(page, '([^#]+)(#?)(.*)')
local text, temp = getText(page, options.noFollow)
if not temp then return throwError('invalid-title', page) end
page = temp
if not text then return throwError('page-not-found', page) end
local full = text -- save the full text for fixReferences below  
-- Get the requested section
if truthy(section) then
text = getSection(text, section)
elseif truthy(hash) then
text = getLead(text)
end  
-- Keep only the requested elements
local elements
if options.only then
if options.only == 'sections' then elements = getSections(text, options.sections) end
if options.only == 'lists' then elements = getLists(text, options.lists) end
if options.only == 'files' then elements = getFiles(text, options.files) end
if options.only == 'tables' then elements = getTables(text, options.tables) end
if options.only == 'templates' then elements = getTemplates(text, options.templates) end
if options.only == 'parameters' then elements = getParameters(text, options.parameters) end
if options.only == 'paragraphs' then elements = getParagraphs(text, options.paragraphs) end
if options.only == 'categories' then elements = getCategories(text, options.categories) end