Update talents + beasts

tools/lpeg/lpcap.lua

@@ -0,0 +1,625 @@
+--[[
+LPEGLJ
+lpcap.lua
+Capture functions
+Copyright (C) 2014 Rostislav Sacek.
+based on LPeg v1.0 - PEG pattern matching for Lua
+Lua.org & PUC-Rio  written by Roberto Ierusalimschy
+http://www.inf.puc-rio.br/~roberto/lpeg/
+
+** Permission is hereby granted, free of charge, to any person obtaining
+** a copy of this software and associated documentation files (the
+** "Software"), to deal in the Software without restriction, including
+** without limitation the rights to use, copy, modify, merge, publish,
+** distribute, sublicense, and/or sell copies of the Software, and to
+** permit persons to whom the Software is furnished to do so, subject to
+** the following conditions:
+**
+** The above copyright notice and this permission notice shall be
+** included in all copies or substantial portions of the Software.
+**
+** THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+** EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+** MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+** IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+** CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+** TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+** SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+**
+** [ MIT license: http://www.opensource.org/licenses/mit-license.php ]
+--]]
+local ffi = require "ffi"
+
+local Cclose = 0
+local Cposition = 1
+local Cconst = 2
+local Cbackref = 3
+local Carg = 4
+local Csimple = 5
+local Ctable = 6
+local Cfunction = 7
+local Cquery = 8
+local Cstring = 9
+local Cnum = 10
+local Csubst = 11
+local Cfold = 12
+local Cruntime = 13
+local Cgroup = 14
+
+local MAXSTRCAPS = 10
+
+local pushcapture
+local addonestring
+
+
+-- Goes back in a list of captures looking for an open capture
+-- corresponding to a close
+
+local function findopen(cs, index)
+    local n = 0; -- number of closes waiting an open
+    while true do
+        index = index - 1
+        if cs.ocap[index].kind == Cclose then
+            n = n + 1 -- one more open to skip
+        elseif cs.ocap[index].siz == 0 then
+            if n == 0 then
+                return index
+            end
+            n = n - 1
+        end
+    end
+end
+
+
+local function checknextcap(cs, captop)
+    local cap = cs.cap;
+    -- not a single capture?    ((cap)->siz != 0)
+    if cs.ocap[cap].siz == 0 then
+        local n = 0; -- number of opens waiting a close
+        -- look for corresponding close
+        while true do
+            cap = cap + 1
+            if cap > captop then return end
+            if cs.ocap[cap].kind == Cclose then
+                n = n - 1
+                if n + 1 == 0 then
+                    break;
+                end
+            elseif cs.ocap[cap].siz == 0 then
+                n = n + 1
+            end
+        end
+    end
+    cap = cap + 1; -- + 1 to skip last close (or entire single capture)
+    if cap > captop then return end
+    return true
+end
+
+
+-- Go to the next capture
+
+local function nextcap(cs)
+    local cap = cs.cap;
+    -- not a single capture?    ((cap)->siz != 0)
+    if cs.ocap[cap].siz == 0 then
+        local n = 0; -- number of opens waiting a close
+        -- look for corresponding close
+        while true do
+            cap = cap + 1
+            if cs.ocap[cap].kind == Cclose then
+                n = n - 1
+                if n + 1 == 0 then
+                    break;
+                end
+            elseif cs.ocap[cap].siz == 0 then
+                n = n + 1
+            end
+        end
+    end
+    cs.cap = cap + 1; -- + 1 to skip last close (or entire single capture)
+end
+
+
+-- Push on the Lua stack all values generated by nested captures inside
+-- the current capture. Returns number of values pushed. 'addextra'
+-- makes it push the entire match after all captured values. The
+-- entire match is pushed also if there are no other nested values,
+-- so the function never returns zero.
+
+local function pushnestedvalues(cs, addextra, out, valuetable)
+    local co = cs.cap
+    cs.cap = cs.cap + 1
+    -- no nested captures?
+    if cs.ocap[cs.cap - 1].siz ~= 0 then
+        local st = cs.ocap[co].s
+        local l = cs.ocap[co].siz - 1
+        out.outindex = out.outindex + 1
+        out.out[out.outindex] = cs.s and cs.s:sub(st, st + l - 1) or cs.stream(st, st + l - 1)
+        return 1; -- that is it
+    else
+        local n = 0;
+        while cs.ocap[cs.cap].kind ~= Cclose do -- repeat for all nested patterns
+        n = n + pushcapture(cs, out, valuetable);
+        end
+        -- need extra?
+        if addextra or n == 0 then
+            local st = cs.ocap[co].s
+            local l = cs.ocap[cs.cap].s - cs.ocap[co].s
+            out.outindex = out.outindex + 1
+            out.out[out.outindex] = cs.s and cs.s:sub(st, st + l - 1) or cs.stream(st, st + l - 1)
+            n = n + 1
+        end
+        cs.cap = cs.cap + 1 -- skip close entry
+        return n;
+    end
+end
+
+
+-- Push only the first value generated by nested captures
+
+local function pushonenestedvalue(cs, out, valuetable)
+    local n = pushnestedvalues(cs, false, out, valuetable)
+    for i = n, 2, -1 do
+        out.out[out.outindex] = nil
+        out.outindex = out.outindex - 1
+    end
+end
+
+
+-- Try to find a named group capture with the name given at the top of
+-- the stack; goes backward from 'cap'.
+
+local function findback(cs, cap, name, valuetable)
+    -- repeat until end of list
+    while cap > 0 do
+        cap = cap - 1
+        local continue
+        if cs.ocap[cap].kind == Cclose then
+            cap = findopen(cs, cap); -- skip nested captures
+        elseif cs.ocap[cap].siz == 0 then
+            continue = true -- opening an enclosing capture: skip and get previous
+        end
+        if not continue and cs.ocap[cap].kind == Cgroup and cs.ocap[cap].idx ~= 0 then
+            local gname = valuetable[cs.ocap[cap].idx] -- get group name
+            -- right group?
+            if name == gname then
+                return cap;
+            end
+        end
+    end
+    error(("back reference '%s' not found"):format(name), 0)
+end
+
+
+-- Back-reference capture. Return number of values pushed.
+
+local function backrefcap(cs, out, valuetable)
+    local curr = cs.cap;
+    local name = valuetable[cs.ocap[cs.cap].idx] -- reference name
+    cs.cap = findback(cs, curr, name, valuetable) -- find corresponding group
+    local n = pushnestedvalues(cs, false, out, valuetable); -- push group's values
+    cs.cap = curr + 1;
+    return n;
+end
+
+
+-- Table capture: creates a new table and populates it with nested
+-- captures.
+
+local function tablecap(cs, out, valuetable)
+    local n = 0;
+    local t = {}
+    cs.cap = cs.cap + 1
+    -- table is empty
+    if cs.ocap[cs.cap - 1].siz == 0 then
+        while cs.ocap[cs.cap].kind ~= Cclose do
+            local subout = { outindex = 0, out = {} }
+            -- named group?
+            if cs.ocap[cs.cap].kind == Cgroup and cs.ocap[cs.cap].idx ~= 0 then
+                local groupname = valuetable[cs.ocap[cs.cap].idx] -- push group name
+                pushonenestedvalue(cs, subout, valuetable)
+                t[groupname] = subout.out[1]
+            else
+                -- not a named group
+                local k = pushcapture(cs, subout, valuetable)
+                -- store all values into table
+                for i = 1, subout.outindex do
+                    t[i + n] = subout.out[i]
+                end
+                n = n + k;
+            end
+        end
+        cs.cap = cs.cap + 1 -- skip close entry
+    end
+    out.outindex = out.outindex + 1
+    out.out[out.outindex] = t
+    return 1; -- number of values pushed (only the table)
+end
+
+
+-- Table-query capture
+
+local function querycap(cs, out, valuetable)
+    local table = valuetable[cs.ocap[cs.cap].idx]
+    local subout = { outindex = 0, out = {} }
+    pushonenestedvalue(cs, subout, valuetable) -- get nested capture
+    -- query cap. value at table
+    if table[subout.out[1]] ~= nil then
+        out.outindex = out.outindex + 1
+        out.out[out.outindex] = table[subout.out[1]]
+        return 1
+    end
+    return 0
+end
+
+
+-- Fold capture
+
+local function foldcap(cs, out, valuetable)
+    local fce = valuetable[cs.ocap[cs.cap].idx]
+    cs.cap = cs.cap + 1
+    -- no nested captures?
+    -- or no nested captures (large subject)?
+    if cs.ocap[cs.cap - 1].siz ~= 0 or
+            cs.ocap[cs.cap].kind == Cclose then
+        error("no initial value for fold capture", 0);
+    end
+    local subout = { outindex = 0; out = {} }
+    local n = pushcapture(cs, subout, valuetable) -- nested captures with no values?
+    if n == 0 then
+        error("no initial value for fold capture", 0);
+    end
+    local acumulator = subout.out[1] -- leave only one result for accumulator
+    while cs.ocap[cs.cap].kind ~= Cclose do
+        local subout = { outindex = 0; out = {} }
+        n = pushcapture(cs, subout, valuetable); -- get next capture's values
+        acumulator = fce(acumulator, unpack(subout.out, 1, subout.outindex)) -- call folding function
+    end
+    cs.cap = cs.cap + 1; -- skip close entry
+    out.outindex = out.outindex + 1
+    out.out[out.outindex] = acumulator
+    return 1; -- only accumulator left on the stack
+end
+
+
+local function retcount(...)
+    return select('#', ...), { ... }
+end
+
+
+-- Function capture
+
+local function functioncap(cs, out, valuetable)
+    local fce = valuetable[cs.ocap[cs.cap].idx] --  push function
+    local subout = { outindex = 0, out = {} }
+    local n = pushnestedvalues(cs, false, subout, valuetable); -- push nested captures
+    local count, ret = retcount(fce(unpack(subout.out, 1, n))) -- call function
+    for i = 1, count do
+        out.outindex = out.outindex + 1
+        out.out[out.outindex] = ret[i]
+    end
+    return count
+end
+
+
+-- Select capture
+
+local function numcap(cs, out, valuetable)
+    local idx = valuetable[cs.ocap[cs.cap].idx] -- value to select
+    -- no values?
+    if idx == 0 then
+        nextcap(cs); -- skip entire capture
+        return 0; -- no value produced
+    else
+        local subout = { outindex = 0, out = {} }
+        local n = pushnestedvalues(cs, false, subout, valuetable)
+        -- invalid index?
+        if n < idx then
+            error(("no capture '%d'"):format(idx), 0)
+        else
+            out.outindex = out.outindex + 1
+            out.out[out.outindex] = subout.out[idx] -- get selected capture
+            return 1;
+        end
+    end
+end
+
+
+-- Calls a runtime capture. Returns number of captures removed by
+-- the call, including the initial Cgroup. (Captures to be added are
+-- on the Lua stack.)
+
+local function runtimecap(cs, close, s, out, valuetable)
+    local open = findopen(cs, close)
+    assert(cs.ocap[open].kind == Cgroup)
+    cs.ocap[close].kind = Cclose; -- closes the group
+    cs.ocap[close].s = s;
+    cs.cap = open;
+    local fce = valuetable[cs.ocap[cs.cap].idx] -- push function to be called
+    local subout = { outindex = 0, out = {} }
+    local n = pushnestedvalues(cs, false, subout, valuetable); -- push nested captures
+    local count, ret = retcount(fce(cs.s or cs.stream, s, unpack(subout.out, 1, n))) -- call dynamic function
+    for i = 1, count do
+        out.outindex = out.outindex + 1
+        out.out[out.outindex] = ret[i]
+    end
+    return close - open -- number of captures of all kinds removed
+end
+
+-- Collect values from current capture into array 'cps'. Current
+-- capture must be Cstring (first call) or Csimple (recursive calls).
+-- (In first call, fills %0 with whole match for Cstring.)
+-- Returns number of elements in the array that were filled.
+
+local function getstrcaps(cs, cps, n)
+    local k = n
+    n = n + 1
+    cps[k + 1].isstring = true; -- get string value
+    cps[k + 1].startstr = cs.ocap[cs.cap].s; -- starts here
+    cs.cap = cs.cap + 1
+    -- nested captures?
+    if cs.ocap[cs.cap - 1].siz == 0 then
+        -- traverse them
+        while cs.ocap[cs.cap].kind ~= Cclose do
+            -- too many captures?
+            if n >= MAXSTRCAPS then
+                nextcap(cs); -- skip extra captures (will not need them)
+            elseif cs.ocap[cs.cap].kind == Csimple then
+                -- string?
+                n = getstrcaps(cs, cps, n); -- put info. into array
+            else
+                cps[n + 1].isstring = false; -- not a string
+                cps[n + 1].origcap = cs.cap; -- keep original capture
+                nextcap(cs);
+                n = n + 1;
+            end
+        end
+        cs.cap = cs.cap + 1 -- skip close
+    end
+    cps[k + 1].endstr = cs.ocap[cs.cap - 1].s + cs.ocap[cs.cap - 1].siz - 1 -- ends here
+    return n;
+end
+
+
+-- add next capture value (which should be a string) to buffer 'b'
+
+-- String capture: add result to buffer 'b' (instead of pushing
+-- it into the stack)
+
+local function stringcap(cs, b, valuetable)
+    local cps = {}
+    for i = 1, MAXSTRCAPS do
+        cps[#cps + 1] = {}
+    end
+    local fmt = valuetable[cs.ocap[cs.cap].idx]
+    local n = getstrcaps(cs, cps, 0) - 1; -- collect nested captures
+    local i = 1
+    -- traverse them
+    while i <= #fmt do
+        local c = fmt:sub(i, i)
+        -- not an escape?
+        if c ~= '%' then
+            b[#b + 1] = c -- add it to buffer
+        elseif fmt:sub(i + 1, i + 1) < '0' or fmt:sub(i + 1, i + 1) > '9' then
+            -- not followed by a digit?
+            i = i + 1
+            b[#b + 1] = fmt:sub(i, i)
+        else
+            i = i + 1
+            local l = fmt:sub(i, i) - '0'; -- capture index
+            if l > n then
+                error(("invalid capture index (%d)"):format(l), 0)
+            elseif cps[l + 1].isstring then
+                b[#b + 1] = cs.s and cs.s:sub(cps[l + 1].startstr, cps[l + 1].endstr - cps[l + 1].startstr + cps[l + 1].startstr - 1) or
+                        cs.stream(cps[l + 1].startstr, cps[l + 1].endstr - cps[l + 1].startstr + cps[l + 1].startstr - 1)
+            else
+                local curr = cs.cap;
+                cs.cap = cps[l + 1].origcap; -- go back to evaluate that nested capture
+                if not addonestring(cs, b, "capture", valuetable) then
+                    error(("no values in capture index %d"):format(l), 0)
+                end
+                cs.cap = curr; -- continue from where it stopped
+            end
+        end
+        i = i + 1
+    end
+end
+
+
+-- Substitution capture: add result to buffer 'b'
+
+local function substcap(cs, b, valuetable)
+    local curr = cs.ocap[cs.cap].s;
+    -- no nested captures?
+    if cs.ocap[cs.cap].siz ~= 0 then
+        -- keep original text
+        b[#b + 1] = cs.s and cs.s:sub(curr, cs.ocap[cs.cap].siz - 1 + curr - 1) or
+                cs.stream(curr, cs.ocap[cs.cap].siz - 1 + curr - 1)
+    else
+        cs.cap = cs.cap + 1 -- skip open entry
+        -- traverse nested captures
+        while cs.ocap[cs.cap].kind ~= Cclose do
+            local next = cs.ocap[cs.cap].s;
+            b[#b + 1] = cs.s and cs.s:sub(curr, next - curr + curr - 1) or
+                    cs.stream(curr, next - curr + curr - 1) -- add text up to capture
+            if addonestring(cs, b, "replacement", valuetable) then
+                curr = cs.ocap[cs.cap - 1].s + cs.ocap[cs.cap - 1].siz - 1; -- continue after match
+            else
+                -- no capture value
+                curr = next; -- keep original text in final result
+            end
+        end
+        b[#b + 1] = cs.s and cs.s:sub(curr, curr + cs.ocap[cs.cap].s - curr - 1) or
+                cs.stream(curr, curr + cs.ocap[cs.cap].s - curr - 1) -- add last piece of text
+    end
+    cs.cap = cs.cap + 1 -- go to next capture
+end
+
+
+-- Evaluates a capture and adds its first value to buffer 'b'; returns
+-- whether there was a value
+
+function addonestring(cs, b, what, valuetable)
+    local tag = cs.ocap[cs.cap].kind
+    if tag == Cstring then
+        stringcap(cs, b, valuetable); -- add capture directly to buffer
+        return 1
+    elseif tag == Csubst then
+        substcap(cs, b, valuetable); -- add capture directly to buffer
+        return 1
+    else
+        local subout = { outindex = 0, out = {} }
+        local n = pushcapture(cs, subout, valuetable);
+        if n > 0 then
+            if type(subout.out[1]) ~= 'string' and type(subout.out[1]) ~= 'number' then
+                error(("invalid %s value (a %s)"):format(what, type(subout.out[1])), 0)
+            end
+            b[#b + 1] = subout.out[1]
+            return n
+        end
+    end
+end
+
+
+-- Push all values of the current capture into the stack; returns
+-- number of values pushed
+
+function pushcapture(cs, out, valuetable)
+    local type = cs.ocap[cs.cap].kind
+    if type == Cposition then
+        out.outindex = out.outindex + 1
+        out.out[out.outindex] = cs.ocap[cs.cap].s
+        cs.cap = cs.cap + 1;
+        return 1;
+    elseif type == Cconst then
+        out.outindex = out.outindex + 1
+        out.out[out.outindex] = valuetable[cs.ocap[cs.cap].idx]
+        cs.cap = cs.cap + 1
+        return 1;
+    elseif type == Carg then
+        local arg = valuetable[cs.ocap[cs.cap].idx]
+        cs.cap = cs.cap + 1
+        if arg > cs.ptopcount then
+            error(("reference to absent extra argument #%d"):format(arg), 0)
+        end
+        out.outindex = out.outindex + 1
+        out.out[out.outindex] = cs.ptop[arg]
+        return 1;
+    elseif type == Csimple then
+        local k = pushnestedvalues(cs, true, out, valuetable)
+        local index = out.outindex
+        table.insert(out.out, index - k + 1, out.out[index])
+        out[index + 1] = nil
+        return k;
+    elseif type == Cruntime then
+        out.outindex = out.outindex + 1
+        out.out[out.outindex] = valuetable[cs.ocap[cs.cap].idx]
+        cs.cap = cs.cap + 1;
+        return 1;
+    elseif type == Cstring then
+        local b = {}
+        stringcap(cs, b, valuetable)
+        out.outindex = out.outindex + 1
+        out.out[out.outindex] = table.concat(b)
+        return 1;
+    elseif type == Csubst then
+        local b = {}
+        substcap(cs, b, valuetable);
+        out.outindex = out.outindex + 1
+        out.out[out.outindex] = table.concat(b)
+        return 1;
+    elseif type == Cgroup then
+        -- anonymous group?
+        if cs.ocap[cs.cap].idx == 0 then
+            return pushnestedvalues(cs, false, out, valuetable); -- add all nested values
+        else
+            -- named group: add no values
+            nextcap(cs); -- skip capture
+            return 0
+        end
+    elseif type == Cbackref then
+        return backrefcap(cs, out, valuetable)
+    elseif type == Ctable then
+        return tablecap(cs, out, valuetable)
+    elseif type == Cfunction then
+        return functioncap(cs, out, valuetable)
+    elseif type == Cnum then
+        return numcap(cs, out, valuetable)
+    elseif type == Cquery then
+        return querycap(cs, out, valuetable)
+    elseif type == Cfold then
+        return foldcap(cs, out, valuetable)
+    else
+        assert(false)
+    end
+end
+
+
+-- Prepare a CapState structure and traverse the entire list of
+-- captures in the stack pushing its results. 's' is the subject
+-- string, 'r' is the final position of the match, and 'ptop'
+-- the index in the stack where some useful values were pushed.
+-- Returns the number of results pushed. (If the list produces no
+-- results, push the final position of the match.)
+
+local function getcaptures(capture, s, stream, r, valuetable, ...)
+    local n = 0;
+    local cs = { cap = 0 }
+    local out = { outindex = 0; out = {} }
+    -- is there any capture?
+    if capture[cs.cap].kind ~= Cclose then
+        cs.ocap = capture
+        cs.s = s;
+        cs.stream = stream
+        cs.ptopcount, cs.ptop = retcount(...)
+        repeat -- collect their values
+            n = n + pushcapture(cs, out, valuetable)
+        until cs.ocap[cs.cap].kind == Cclose
+    end
+    -- no capture values?
+    if n == 0 then
+        if not r then
+            return
+        else
+            return r
+        end
+    end
+    assert(out.outindex < 7998, "(too many captures)")
+    return unpack(out.out, 1, out.outindex)
+end
+
+local function getcapturesruntime(capture, s, stream, notdelete, min, max, captop, valuetable, ...)
+    local n = 0;
+    local cs = { cap = min }
+    local out = { outindex = 0; out = {} }
+    cs.ocap = capture
+    cs.s = s
+    cs.stream = stream
+    cs.ptopcount, cs.ptop = retcount(...)
+    local start = 0
+    repeat -- collect their values
+        if not checknextcap(cs, max) then break end
+        local notdelete = notdelete or capture[cs.cap].kind == Cgroup and capture[cs.cap].idx ~= 0 and capture[cs.cap].candelete == 0
+        pushcapture(cs, out, valuetable)
+        if notdelete then
+            start = cs.cap
+        else
+            n = n + cs.cap - start
+            for i = 0, captop - cs.cap - 1 do
+                ffi.copy(capture + start + i, capture + cs.cap + i, ffi.sizeof('CAPTURE'))
+            end
+            max = max - (cs.cap - start)
+            captop = captop - (cs.cap - start)
+            cs.cap = start
+        end
+    until cs.cap == max
+    assert(out.outindex < 7998, "(too many captures)")
+    return n, out.out, out.outindex
+end
+
+return {
+    getcaptures = getcaptures,
+    runtimecap = runtimecap,
+    getcapturesruntime = getcapturesruntime,
+}
+

tools/lpeg/lpprint.lua

@@ -0,0 +1,356 @@
+--[[
+LPEGLJ
+lpprint.lua
+Tree, code and debug print function (only for debuging)
+Copyright (C) 2014 Rostislav Sacek.
+based on LPeg v1.0 - PEG pattern matching for Lua
+Lua.org & PUC-Rio  written by Roberto Ierusalimschy
+http://www.inf.puc-rio.br/~roberto/lpeg/
+
+** Permission is hereby granted, free of charge, to any person obtaining
+** a copy of this software and associated documentation files (the
+** "Software"), to deal in the Software without restriction, including
+** without limitation the rights to use, copy, modify, merge, publish,
+** distribute, sublicense, and/or sell copies of the Software, and to
+** permit persons to whom the Software is furnished to do so, subject to
+** the following conditions:
+**
+** The above copyright notice and this permission notice shall be
+** included in all copies or substantial portions of the Software.
+**
+** THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+** EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+** MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+** IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+** CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+** TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+** SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+**
+** [ MIT license: http://www.opensource.org/licenses/mit-license.php ]
+--]]
+
+local ffi = require"ffi"
+local band, rshift, lshift = bit.band, bit.rshift, bit.lshift
+
+ffi.cdef[[
+  int isprint ( int c );
+]]
+
+local RuleLR = 0x10000
+local Ruleused = 0x20000
+
+-- {======================================================
+-- Printing patterns (for debugging)
+-- =======================================================
+
+local TChar = 0
+local TSet = 1
+local TAny = 2 -- standard PEG elements
+local TTrue = 3
+local TFalse = 4
+local TRep = 5
+local TSeq = 6
+local TChoice = 7
+local TNot = 8
+local TAnd = 9
+local TCall = 10
+local TOpenCall = 11
+local TRule = 12 -- sib1 is rule's pattern, sib2 is 'next' rule
+local TGrammar = 13 -- sib1 is initial (and first) rule
+local TBehind = 14 -- match behind
+local TCapture = 15 -- regular capture
+local TRunTime = 16 -- run-time capture
+
+local IAny = 0 -- if no char, fail
+local IChar = 1 -- if char != aux, fail
+local ISet = 2 -- if char not in val, fail
+local ITestAny = 3 -- in no char, jump to 'offset'
+local ITestChar = 4 -- if char != aux, jump to 'offset'
+local ITestSet = 5 -- if char not in val, jump to 'offset'
+local ISpan = 6 -- read a span of chars in val
+local IBehind = 7 -- walk back 'aux' characters (fail if not possible)
+local IRet = 8 -- return from a rule
+local IEnd = 9 -- end of pattern
+local IChoice = 10 -- stack a choice; next fail will jump to 'offset'
+local IJmp = 11 -- jump to 'offset'
+local ICall = 12 -- call rule at 'offset'
+local IOpenCall = 13 -- call rule number 'offset' (must be closed to a ICall)
+local ICommit = 14 -- pop choice and jump to 'offset'
+local IPartialCommit = 15 -- update top choice to current position and jump
+local IBackCommit = 16 -- "fails" but jump to its own 'offset'
+local IFailTwice = 17 -- pop one choice and then fail
+local IFail = 18 -- go back to saved state on choice and jump to saved offset
+local IGiveup = 19 -- internal use
+local IFullCapture = 20 -- complete capture of last 'off' chars
+local IOpenCapture = 21 -- start a capture
+local ICloseCapture = 22
+local ICloseRunTime = 23
+
+local Cclose = 0
+local Cposition = 1
+local Cconst = 2
+local Cbackref = 3
+local Carg = 4
+local Csimple = 5
+local Ctable = 6
+local Cfunction = 7
+local Cquery = 8
+local Cstring = 9
+local Cnum = 10
+local Csubst = 11
+local Cfold = 12
+local Cruntime = 13
+local Cgroup = 14
+
+
+-- number of siblings for each tree
+local numsiblings = {
+    [TRep] = 1,
+    [TSeq] = 2,
+    [TChoice] = 2,
+    [TNot] = 1,
+    [TAnd] = 1,
+    [TRule] = 2,
+    [TGrammar] = 1,
+    [TBehind] = 1,
+    [TCapture] = 1,
+    [TRunTime] = 1,
+}
+local names = {
+    [IAny] = "any",
+    [IChar] = "char",
+    [ISet] = "set",
+    [ITestAny] = "testany",
+    [ITestChar] = "testchar",
+    [ITestSet] = "testset",
+    [ISpan] = "span",
+    [IBehind] = "behind",
+    [IRet] = "ret",
+    [IEnd] = "end",
+    [IChoice] = "choice",
+    [IJmp] = "jmp",
+    [ICall] = "call",
+    [IOpenCall] = "open_call",
+    [ICommit] = "commit",
+    [IPartialCommit] = "partial_commit",
+    [IBackCommit] = "back_commit",
+    [IFailTwice] = "failtwice",
+    [IFail] = "fail",
+    [IGiveup] = "giveup",
+    [IFullCapture] = "fullcapture",
+    [IOpenCapture] = "opencapture",
+    [ICloseCapture] = "closecapture",
+    [ICloseRunTime] = "closeruntime"
+}
+
+local function printcharset(st)
+    io.write("[");
+    local i = 0
+    while i <= 255 do
+        local first = i;
+        while band(st[rshift(i, 5)], lshift(1, band(i, 31))) ~= 0 and i <= 255 do
+            i = i + 1
+        end
+        if i - 1 == first then -- unary range?
+            io.write(("(%02x)"):format(first))
+        elseif i - 1 > first then -- non-empty range?
+            io.write(("(%02x-%02x)"):format(first, i - 1))
+        end
+        i = i + 1
+    end
+    io.write("]")
+end
+
+local modes = {
+    [Cclose] = "close",
+    [Cposition] = "position",
+    [Cconst] = "constant",
+    [Cbackref] = "backref",
+    [Carg] = "argument",
+    [Csimple] = "simple",
+    [Ctable] = "table",
+    [Cfunction] = "function",
+    [Cquery] = "query",
+    [Cstring] = "string",
+    [Cnum] = "num",
+    [Csubst] = "substitution",
+    [Cfold] = "fold",
+    [Cruntime] = "runtime",
+    [Cgroup] = "group"
+}
+
+local function printcapkind(kind)
+    io.write(("%s"):format(modes[kind]))
+end
+
+local function printjmp(p, index)
+    io.write(("-> %d"):format(index + p[index].offset))
+end
+
+local function printrulename(p, index, rulenames)
+    if rulenames and rulenames[index + p[index].offset] then
+        io.write(' ', rulenames[index + p[index].offset])
+    end
+end
+
+local function printinst(p, index, valuetable, rulenames)
+    local code = p[index].code
+    if rulenames and rulenames[index] then
+        io.write(rulenames[index], '\n')
+    end
+    io.write(("%04d: %s "):format(index, names[code]))
+    if code == IChar then
+        io.write(("'%s'"):format(string.char(p[index].val)))
+    elseif code == ITestChar then
+        io.write(("'%s'"):format(string.char(p[index].val)))
+        printjmp(p, index)
+        printrulename(p, index, rulenames)
+    elseif code == IFullCapture then
+        printcapkind(band(p[index].val, 0x0f));
+        io.write((" (size = %d)  (idx = %s)"):format(band(rshift(p[index].val, 4), 0xF), tostring(valuetable[p[index].offset])))
+    elseif code == IOpenCapture then
+        printcapkind(band(p[index].val, 0x0f))
+        io.write((" (idx = %s)"):format(tostring(valuetable[p[index].offset])))
+    elseif code == ISet then
+        printcharset(valuetable[p[index].val]);
+    elseif code == ITestSet then
+        printcharset(valuetable[p[index].val])
+        printjmp(p, index);
+        printrulename(p, index, rulenames)
+    elseif code == ISpan then
+        printcharset(valuetable[p[index].val]);
+    elseif code == IOpenCall then
+        io.write(("-> %d"):format(p[index].offset))
+    elseif code == IBehind then
+        io.write(("%d"):format(p[index].val))
+    elseif code == IJmp or code == ICall or code == ICommit or code == IChoice or
+            code == IPartialCommit or code == IBackCommit or code == ITestAny then
+        printjmp(p, index);
+        if (code == ICall or code == IJmp) and p[index].aux > 0 then
+            io.write(' ', valuetable[p[index].aux])
+        else
+            printrulename(p, index, rulenames)
+        end
+    end
+    io.write("\n")
+end
+
+
+local function printpatt(p, valuetable)
+    local ruleNames = {}
+    for i = 0, p.size - 1 do
+        local code = p.p[i].code
+        if (code == ICall or code == IJmp) and p.p[i].aux > 0 then
+            local index = i + p.p[i].offset
+            ruleNames[index] = valuetable[p.p[i].aux]
+        end
+    end
+    for i = 0, p.size - 1 do
+        printinst(p.p, i, valuetable, ruleNames)
+    end
+end
+
+
+local function printcap(cap, index, valuetable)
+    printcapkind(cap[index].kind)
+    io.write((" (idx: %s - size: %d) -> %d\n"):format(valuetable[cap[index].idx], cap[index].siz, cap[index].s))
+end
+
+
+local function printcaplist(cap, limit, valuetable)
+    io.write(">======\n")
+    local index = 0
+    while cap[index].s and index < limit do
+        printcap(cap, index, valuetable)
+        index = index + 1
+    end
+    io.write("=======\n")
+end
+
+-- ======================================================
+
+
+
+-- {======================================================
+-- Printing trees (for debugging)
+-- =======================================================
+
+local tagnames = {
+    [TChar] = "char",
+    [TSet] = "set",
+    [TAny] = "any",
+    [TTrue] = "true",
+    [TFalse] = "false",
+    [TRep] = "rep",
+    [TSeq] = "seq",
+    [TChoice] = "choice",
+    [TNot] = "not",
+    [TAnd] = "and",
+    [TCall] = "call",
+    [TOpenCall] = "opencall",
+    [TRule] = "rule",
+    [TGrammar] = "grammar",
+    [TBehind] = "behind",
+    [TCapture] = "capture",
+    [TRunTime] = "run-time"
+}
+
+
+local function printtree(tree, ident, index, valuetable)
+    for i = 1, ident do
+        io.write(" ")
+    end
+    local tag = tree[index].tag
+    io.write(("%s"):format(tagnames[tag]))
+    if tag == TChar then
+        local c = tree[index].val
+        if ffi.C.isprint(c) then
+            io.write((" '%c'\n"):format(c))
+        else
+            io.write((" (%02X)\n"):format(c))
+        end
+    elseif tag == TSet then
+        printcharset(valuetable[tree[index].val]);
+        io.write("\n")
+    elseif tag == TOpenCall or tag == TCall then
+        io.write((" key: %s\n"):format(tostring(valuetable[tree[index].val])))
+    elseif tag == TBehind then
+        io.write((" %d\n"):format(tree[index].val))
+        printtree(tree, ident + 2, index + 1, valuetable);
+    elseif tag == TCapture then
+        io.write((" cap: %s   n: %s\n"):format(modes[bit.band(tree[index].cap, 0xffff)], valuetable[tree[index].val]))
+        printtree(tree, ident + 2, index + 1, valuetable);
+    elseif tag == TRule then
+        local extra = bit.band(tree[index].cap, RuleLR) == RuleLR and ' left recursive' or ''
+        extra = extra .. (bit.band(tree[index].cap, Ruleused) ~= Ruleused and ' not used' or '')
+        io.write((" n: %d  key: %s%s\n"):format(bit.band(tree[index].cap, 0xffff) - 1, valuetable[tree[index].val], extra))
+        printtree(tree, ident + 2, index + 1, valuetable);
+        -- do not print next rule as a sibling
+    elseif tag == TGrammar then
+        local ruleindex = index + 1
+        io.write((" %d\n"):format(tree[index].val)) -- number of rules
+        for i = 1, tree[index].val do
+            printtree(tree, ident + 2, ruleindex, valuetable);
+            ruleindex = ruleindex + tree[ruleindex].ps
+        end
+        assert(tree[ruleindex].tag == TTrue); -- sentinel
+    else
+        local sibs = numsiblings[tree[index].tag] or 0
+        io.write("\n")
+        if sibs >= 1 then
+            printtree(tree, ident + 2, index + 1, valuetable);
+            if sibs >= 2 then
+                printtree(tree, ident + 2, index + tree[index].ps, valuetable)
+            end
+        end
+    end
+end
+
+-- }====================================================== */
+
+return {
+    printtree = printtree,
+    printpatt = printpatt,
+    printcaplist = printcaplist,
+    printinst = printinst
+}

tools/lpeg/re.lua

@@ -0,0 +1,286 @@
+-- $Id: re.lua,v 1.44 2013/03/26 20:11:40 roberto Exp $
+-- 2014/08/15 changes rostislav
+
+-- imported functions and modules
+local tonumber, print, error = tonumber, print, error
+local setmetatable = setmetatable
+local m = require"lpeglj"
+
+-- 'm' will be used to parse expressions, and 'mm' will be used to
+-- create expressions; that is, 're' runs on 'm', creating patterns
+-- on 'mm'
+local mm = m
+
+-- pattern's metatable
+local mt = getmetatable(mm.P(0))
+mt = m.version() == "1.0.0.0LJ" and m or mt
+
+
+
+-- No more global accesses after this point
+local version = _VERSION
+if version == "Lua 5.2" then _ENV = nil end
+
+
+local any = m.P(1)
+
+
+-- Pre-defined names
+local Predef = { nl = m.P"\n" }
+
+
+local mem
+local fmem
+local gmem
+
+
+local function updatelocale ()
+  mm.locale(Predef)
+  Predef.a = Predef.alpha
+  Predef.c = Predef.cntrl
+  Predef.d = Predef.digit
+  Predef.g = Predef.graph
+  Predef.l = Predef.lower
+  Predef.p = Predef.punct
+  Predef.s = Predef.space
+  Predef.u = Predef.upper
+  Predef.w = Predef.alnum
+  Predef.x = Predef.xdigit
+  Predef.A = any - Predef.a
+  Predef.C = any - Predef.c
+  Predef.D = any - Predef.d
+  Predef.G = any - Predef.g
+  Predef.L = any - Predef.l
+  Predef.P = any - Predef.p
+  Predef.S = any - Predef.s
+  Predef.U = any - Predef.u
+  Predef.W = any - Predef.w
+  Predef.X = any - Predef.x
+  mem = {}    -- restart memoization
+  fmem = {}
+  gmem = {}
+  local mt = {__mode = "v"}
+  setmetatable(mem, mt)
+  setmetatable(fmem, mt)
+  setmetatable(gmem, mt)
+end
+
+
+updatelocale()
+
+
+
+local I = m.P(function (s,i) print(i, s:sub(1, i-1)); return i end)
+
+
+local function getdef (id, defs)
+  local c = defs and defs[id]
+  if not c then error("undefined name: " .. id) end
+  return c
+end
+
+
+local function patt_error (s, i)
+  local msg = (#s < i + 20) and s:sub(i)
+                             or s:sub(i,i+20) .. "..."
+  msg = ("pattern error near '%s'"):format(msg)
+  error(msg, 2)
+end
+
+local function mult (p, n)
+  local np = mm.P(true)
+  while n >= 1 do
+    if n%2 >= 1 then np = np * p end
+    p = p * p
+    n = n/2
+  end
+  return np
+end
+
+local function equalcap (s, i, c)
+  if type(c) ~= "string" then return nil end
+  local e = #c + i
+  if type(s) == 'function' then  -- stream mode
+      if s(i, e - 1) == c then return e else return nil end
+  else
+      if s:sub(i, e - 1) == c then return e else return nil end
+  end
+end
+
+
+local S = (Predef.space + "--" * (any - Predef.nl)^0)^0
+
+local name = m.R("AZ", "az", "__") * m.R("AZ", "az", "__", "09")^0
+
+local arrow = S * "<-"
+
+local seq_follow = m.P"/" + ")" + "}" + ":}" + "~}" + "|}" + (name * arrow) + -1
+
+name = m.C(name)
+
+
+-- a defined name only have meaning in a given environment
+local Def = name * m.Carg(1)
+
+local num = m.C(m.R"09"^1) * S / tonumber
+
+local String = "'" * m.C((any - "'")^0) * "'" +
+               '"' * m.C((any - '"')^0) * '"'
+
+
+local defined = "%" * Def / function (c,Defs)
+  local cat =  Defs and Defs[c] or Predef[c]
+  if not cat then error ("name '" .. c .. "' undefined") end
+  return cat
+end
+
+local Range = m.Cs(any * (m.P"-"/"") * (any - "]")) / mm.R
+
+local item = defined + Range + m.C(any)
+
+local Class =
+    "["
+  * (m.C(m.P"^"^-1))    -- optional complement symbol
+  * m.Cf(item * (item - "]")^0, mt.__add) /
+                          function (c, p) return c == "^" and any - p or p end
+  * "]"
+
+local function adddef (t, k, exp)
+  if t[k] then
+    error("'"..k.."' already defined as a rule")
+  else
+    t[k] = exp
+  end
+  return t
+end
+
+local function firstdef (n, r) return adddef({n}, n, r) end
+
+
+local function NT (n, b, p)
+  if not b then
+    error("rule '"..n.."' used outside a grammar")
+  else return mm.V(n, p or 0)
+  end
+end
+
+
+local exp = m.P{ "Exp",
+  Exp = S * ( m.V"Grammar"
+            + m.Cf(m.V"Seq" * ("/" * S * m.V"Seq")^0, mt.__add) );
+  Seq = m.Cf(m.Cc(m.P"") * m.V"Prefix"^0 , mt.__mul)
+        * (#seq_follow + patt_error);
+  Prefix = "&" * S * m.V"Prefix" / mt.__len
+         + "!" * S * m.V"Prefix" / mt.__unm
+         + m.V"Suffix";
+  Suffix = m.Cf(m.V"Primary" * S *
+          ( ( m.P"+" * m.Cc(1, mt.__pow)
+            + m.P"*" * m.Cc(0, mt.__pow)
+            + m.P"?" * m.Cc(-1, mt.__pow)
+            + "^" * ( m.Cg(num * m.Cc(mult))
+                    + m.Cg(m.C(m.S"+-" * m.R"09"^1) * m.Cc(mt.__pow))
+                    )
+            + "->" * S * ( m.Cg((String + num) * m.Cc(mt.__div))
+                         + m.P"{}" * m.Cc(nil, m.Ct)
+                         + m.Cg(Def / getdef * m.Cc(mt.__div))
+                         )
+            + "=>" * S * m.Cg(Def / getdef * m.Cc(m.Cmt))
+            ) * S
+          )^0, function (a,b,f) return f(a,b) end );
+  Primary = "(" * m.V"Exp" * ")"
+            + String / mm.P
+            + Class
+            + defined
+            + "{:" * (name * ":" + m.Cc(nil)) * m.V"Exp" * ":}" /
+                     function (n, p) return mm.Cg(p, n) end
+            + "=" * name / function (n) return mm.Cmt(mm.Cb(n), equalcap) end
+            + m.P"{}" / mm.Cp
+            + "{~" * m.V"Exp" * "~}" / mm.Cs
+            + "{|" * m.V"Exp" * "|}" / mm.Ct
+            + "{" * m.V"Exp" * "}" / mm.C
+            + m.P"." * m.Cc(any)
+            + (name * m.Cb("G") * (S * ":" * S * num)^-1 * -arrow + "<" * name * m.Cb("G") * (S * ":" * S * num)^-1 * ">") / NT;
+  Definition = name * arrow * m.V"Exp";
+  Grammar = m.Cg(m.Cc(true), "G") *
+            m.Cf(m.V"Definition" / firstdef * m.Cg(m.V"Definition")^0,
+              adddef) / mm.P
+}
+
+local pattern = S * m.Cg(m.Cc(false), "G") * exp / mm.P * (-any + patt_error)
+
+
+local function compile (p, defs)
+  if mm.type(p) == "pattern" then return p end   -- already compiled
+  local cp = pattern:match(p, 1, defs)
+  if not cp then error("incorrect pattern", 3) end
+  return cp
+end
+
+local function match (s, p, i)
+  local cp = mem[p]
+  if not cp then
+    cp = compile(p)
+    mem[p] = cp
+  end
+  return cp:match(s, i or 1)
+end
+
+local function streammatch (p, i)
+    local cp = mem[p]
+    if not cp then
+        cp = compile(p)
+        mem[p] = cp
+    end
+    return cp:streammatch(i or 1)
+end
+
+-- Only for testing purpose
+local function emulatestreammatch(s, p, i)
+    local cp = mem[p]
+    if not cp then
+        cp = compile(p)
+        mem[p] = cp
+    end
+    return cp:emulatestreammatch(s, i or 1)
+end
+
+local function find (s, p, i)
+  local cp = fmem[p]
+  if not cp then
+    cp = compile(p) / 0
+    cp = mm.P{ mm.Cp() * cp * mm.Cp() + 1 * mm.V(1) }
+    fmem[p] = cp
+  end
+  local i, e = cp:match(s, i or 1)
+  if i then return i, e - 1
+  else return i
+  end
+end
+
+local function gsub (s, p, rep)
+  local g = gmem[p] or {}   -- ensure gmem[p] is not collected while here
+  gmem[p] = g
+  local cp = g[rep]
+  if not cp then
+    cp = compile(p)
+    cp = mm.Cs((cp / rep + 1)^0)
+    g[rep] = cp
+  end
+  return cp:match(s)
+end
+
+
+-- exported names
+local re = {
+  compile = compile,
+  match = match,
+  streammatch = streammatch,
+  emulatestreammatch = emulatestreammatch,
+  find = find,
+  gsub = gsub,
+  updatelocale = updatelocale,
+}
+
+if version == "Lua 5.1" then _G.re = re end
+
+return re