Skip to content

Commit d52a5e2

Browse files
ReFreezedReFreezed
committed
Added support for LuaJIT-specific syntax. Output validation can now be disabled. Fixed parsing of &, | and ~ operators.
1 parent 03bb4e1 commit d52a5e2

File tree

2 files changed

+82
-40
lines changed

2 files changed

+82
-40
lines changed

‎preprocess-cl.lua‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,11 @@ exec lua "$0" "$@"
4747
(See 'Handler messages' and misc/testHandler.lua)
4848
The file shares the same environment as the processed files.
4949
50+
--jitsyntax
51+
Allow LuaJIT-specific syntax, specifically literals for 64-bit
52+
integers and complex numbers.
53+
(https://luajit.org/ext_ffi_api.html#literals)
54+
5055
--linenumbers
5156
Add comments with line numbers to the output.
5257
@@ -58,6 +63,9 @@ exec lua "$0" "$@"
5863
--nonil
5964
Disallow !(...) and outputValue(...) from outputting nil.
6065
66+
--novalidate
67+
Disable validation of outputted Lua.
68+
6169
--outputextension=fileExtension
6270
Specify what file extension generated files should have. The
6371
default is "lua". If any input files end in .lua then you must
@@ -154,6 +162,7 @@ local pp = dofile((args[0]:gsub("[^/\\]+$", "preprocess.lua")))
154162
-- From args:
155163
local addLineNumbers = false
156164
local allowBacktickStrings = false
165+
local allowJitSyntax = false
157166
local canOutputNil = true
158167
local customData = nil
159168
local hasOutputExtension = false
@@ -163,6 +172,7 @@ local outputExtension = "lua"
163172
local outputMeta = false
164173
local processingInfoPath = ""
165174
local silent = false
175+
local validate = true
166176

167177
--==============================================================
168178
--= Local Functions ============================================
@@ -256,6 +266,9 @@ for _, arg in ipairs(args) do
256266
elseif arg:find"^%-%-handler=" or arg:find"^%-h=" then
257267
messageHandlerPath = arg:match"^%-%-handler=(.*)$" or arg:match"^%-h=(.*)$"
258268

269+
elseif arg == "--jitsyntax" then
270+
allowJitSyntax = true
271+
259272
elseif arg == "--linenumbers" then
260273
addLineNumbers = true
261274

@@ -265,6 +278,9 @@ for _, arg in ipairs(args) do
265278
elseif arg == "--nonil" then
266279
canOutputNil = false
267280

281+
elseif arg == "--novalidate" then
282+
validate = false
283+
268284
elseif arg:find"^%-%-outputextension=" then
269285
if hasOutputPaths then
270286
errorline("Cannot specify both --outputextension and --outputpaths")
@@ -438,6 +454,8 @@ for i, pathIn in ipairs(pathsIn) do
438454

439455
backtickStrings = allowBacktickStrings,
440456
canOutputNil = canOutputNil,
457+
jitSyntax = allowJitSyntax,
458+
validate = validate,
441459

442460
onInsert = (hasMessageHandler("insert") or nil) and function(name)
443461
local lua = sendMessage("insert", pathIn, name)

‎preprocess.lua‎

Lines changed: 64 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -198,8 +198,9 @@ local canOutputNil = true
198198
--==============================================================
199199
--= Local Functions ============================================
200200
--==============================================================
201+
local _concatTokens
202+
local _tokenize
201203
local assertarg
202-
local concatTokens
203204
local copyTable
204205
local countString
205206
local error, errorline, errorOnLine, errorInFile, errorAtToken
@@ -218,7 +219,6 @@ local pack, unpack
218219
local printf, printTokens, printTraceback
219220
local pushErrorHandler, pushErrorHandlerIfOverridingDefault, popErrorHandler
220221
local serialize, toLua
221-
local tokenize
222222

223223
F = string.format
224224
function tryToFormatError(err0)
@@ -385,8 +385,7 @@ local NUM_DEC_FRAC = ("^( %d* %.%d+
385385
local NUM_DEC_EXP = ("^( %d+ %.? [Ee][-+]?%d+ )"):gsub(" +", "")
386386
local NUM_DEC = ("^( %d+ %.? )"):gsub(" +", "")
387387

388-
-- tokens = tokenize( lua, filePath, allowBacktickStrings [, allowPreprocessorTokens=false ] )
389-
function tokenize(s, path, allowBacktickStrings, allowMetaTokens)
388+
function _tokenize(s, path, allowMetaTokens, allowBacktickStrings, allowJitSyntax)
390389
local tokens = {}
391390
local ptr = 1
392391
local ln = 1
@@ -415,31 +414,48 @@ function tokenize(s, path, allowBacktickStrings, allowMetaTokens)
415414

416415
-- Number.
417416
elseif s:find("^%.?%d", ptr) then
418-
local lua52Hex, i1, i2, numStr = true, s:find(NUM_HEX_FRAC_EXP, ptr)
419-
if not i1 then lua52Hex, i1, i2, numStr = true, s:find(NUM_HEX_FRAC, ptr) end
420-
if not i1 then lua52Hex, i1, i2, numStr = true, s:find(NUM_HEX_EXP, ptr) end
421-
if not i1 then lua52Hex, i1, i2, numStr = false, s:find(NUM_HEX, ptr) end
422-
if not i1 then lua52Hex, i1, i2, numStr = false, s:find(NUM_DEC_FRAC_EXP, ptr) end
423-
if not i1 then lua52Hex, i1, i2, numStr = false, s:find(NUM_DEC_FRAC, ptr) end
424-
if not i1 then lua52Hex, i1, i2, numStr = false, s:find(NUM_DEC_EXP, ptr) end
425-
if not i1 then lua52Hex, i1, i2, numStr = false, s:find(NUM_DEC, ptr) end
417+
local pat, maybeInt, lua52Hex, i1, i2, numStr = NUM_HEX_FRAC_EXP, false, true, s:find(NUM_HEX_FRAC_EXP, ptr)
418+
if not i1 then pat, maybeInt, lua52Hex, i1, i2, numStr = NUM_HEX_FRAC, false, true, s:find(NUM_HEX_FRAC, ptr)
419+
if not i1 then pat, maybeInt, lua52Hex, i1, i2, numStr = NUM_HEX_EXP, false, true, s:find(NUM_HEX_EXP, ptr)
420+
if not i1 then pat, maybeInt, lua52Hex, i1, i2, numStr = NUM_HEX, true, false, s:find(NUM_HEX, ptr)
421+
if not i1 then pat, maybeInt, lua52Hex, i1, i2, numStr = NUM_DEC_FRAC_EXP, false, false, s:find(NUM_DEC_FRAC_EXP, ptr)
422+
if not i1 then pat, maybeInt, lua52Hex, i1, i2, numStr = NUM_DEC_FRAC, false, false, s:find(NUM_DEC_FRAC, ptr)
423+
if not i1 then pat, maybeInt, lua52Hex, i1, i2, numStr = NUM_DEC_EXP, false, false, s:find(NUM_DEC_EXP, ptr)
424+
if not i1 then pat, maybeInt, lua52Hex, i1, i2, numStr = NUM_DEC, true, false, s:find(NUM_DEC, ptr)
425+
end end end end end end end
426426

427427
if not numStr then
428428
return nil, errorInFile(s, path, ptr, "Tokenizer", "Malformed number.")
429429
end
430-
if s:find("^[%w_]", i2+1) then
431-
-- This is actually only an error in Lua 5.1. Maybe we should issue a warning instead of an error here?
432-
return nil, errorInFile(s, path, i2+1, "Tokenizer", "Malformed number.")
430+
431+
local numStrFallback = numStr
432+
433+
if allowJitSyntax then
434+
if s:find("^[Ii]", i2+1) then -- Imaginary part of complex number.
435+
numStr = s:sub(i1, i2+1)
436+
i2 = i2 + 1
437+
438+
elseif not maybeInt or numStr:find(".", 1, true) then
439+
-- void
440+
elseif s:find("^[Uu][Ll][Ll]", i2+1) then -- Unsigned 64-bit integer.
441+
numStr = s:sub(i1, i2+3)
442+
i2 = i2 + 3
443+
elseif s:find("^[Ll][Ll]", i2+1) then -- Signed 64-bit integer.
444+
numStr = s:sub(i1, i2+2)
445+
i2 = i2 + 2
446+
end
433447
end
434448

435-
local n = tonumber(numStr)
449+
local n = tonumber(numStr) or tonumber(numStrFallback)
436450

437451
-- Support hexadecimal floats in Lua 5.1.
438452
if not n and lua52Hex then
439-
local _, intStr, fracStr, expStr = numStr:match(NUM_HEX_FRAC_EXP)
440-
if not intStr then _, intStr, fracStr = numStr:match(NUM_HEX_FRAC) ; expStr = "0" end
441-
if not intStr then _, intStr, expStr = numStr:match(NUM_HEX_EXP) ; fracStr = "" end
442-
assert(intStr, numStr)
453+
-- Note: We know we're not running LuaJIT here as it supports hexadecimal floats, thus we use numStrFallback instead of numStr.
454+
local _, intStr, fracStr, expStr
455+
if pat == NUM_HEX_FRAC_EXP then _, intStr, fracStr, expStr = numStrFallback:match(NUM_HEX_FRAC_EXP)
456+
elseif pat == NUM_HEX_FRAC then _, intStr, fracStr = numStrFallback:match(NUM_HEX_FRAC) ; expStr = "0"
457+
elseif pat == NUM_HEX_EXP then _, intStr, expStr = numStrFallback:match(NUM_HEX_EXP) ; fracStr = ""
458+
else assert(false) end
443459

444460
n = tonumber(intStr, 16) or 0 -- intStr may be "".
445461

@@ -456,6 +472,11 @@ function tokenize(s, path, allowBacktickStrings, allowMetaTokens)
456472
return nil, errorInFile(s, path, ptr, "Tokenizer", "Invalid number.")
457473
end
458474

475+
if s:find("^[%w_]", i2+1) then
476+
-- This is actually only an error in Lua 5.1. Maybe we should issue a warning instead of an error here?
477+
return nil, errorInFile(s, path, i2+1, "Tokenizer", "Malformed number.")
478+
end
479+
459480
ptr = i2+1
460481
tok = {type="number", representation=numStr, value=n}
461482

@@ -592,15 +613,15 @@ function tokenize(s, path, allowBacktickStrings, allowMetaTokens)
592613
tok = {type="string", representation=repr, value=v, long=false}
593614

594615
-- Punctuation etc.
595-
elseif s:find("^%.%.%.", ptr) then
616+
elseif s:find("^%.%.%.", ptr) then -- 3
596617
local repr = s:sub(ptr, ptr+2)
597618
tok = {type="punctuation", representation=repr, value=repr}
598619
ptr = ptr+#repr
599-
elseif s:find("^%.%.", ptr) or s:find("^[=~<>]=", ptr) or s:find("^::", ptr) or s:find("^//", ptr) or s:find("^<<", ptr) or s:find("^>>", ptr) then
620+
elseif s:find("^%.%.", ptr) or s:find("^[=~<>]=", ptr) or s:find("^::", ptr) or s:find("^//", ptr) or s:find("^<<", ptr) or s:find("^>>", ptr) then -- 2
600621
local repr = s:sub(ptr, ptr+1)
601622
tok = {type="punctuation", representation=repr, value=repr}
602623
ptr = ptr+#repr
603-
elseif s:find("^[+%-*/%%^#<>=(){}[%];:,.]", ptr) then
624+
elseif s:find("^[+%-*/%%^#<>=(){}[%];:,.&|~]", ptr) then -- 1
604625
local repr = s:sub(ptr, ptr)
605626
tok = {type="punctuation", representation=repr, value=repr}
606627
ptr = ptr+#repr
@@ -636,7 +657,7 @@ function tokenize(s, path, allowBacktickStrings, allowMetaTokens)
636657
return tokens
637658
end
638659

639-
function concatTokens(tokens, lastLn, addLineNumbers)
660+
function _concatTokens(tokens, lastLn, addLineNumbers)
640661
local parts = {}
641662

642663
if addLineNumbers then
@@ -1157,7 +1178,7 @@ end
11571178
-- }
11581179
function metaFuncs.tokenize(lua, allowMetaTokens)
11591180
pushErrorHandler(NOOP)
1160-
local tokens, err = tokenize(lua, "<string>", allowMetaTokens, allowMetaTokens)
1181+
local tokens, err = _tokenize(lua, "<string>", allowMetaTokens, allowMetaTokens, true) -- @Incomplete: Make allowJitSyntax a parameter to tokenize()?
11611182
popErrorHandler()
11621183
return tokens, err
11631184
end
@@ -1177,7 +1198,7 @@ function metaFuncs.removeUselessTokens(tokens)
11771198
end
11781199
end
11791200

1180-
for i = len+offset+1, len do
1201+
for i = len, len+offset+1, -1 do
11811202
tokens[i] = nil
11821203
end
11831204
end
@@ -1398,7 +1419,7 @@ end
13981419
-- Concatinate tokens by their representations.
13991420
-- luaString = concatTokens( tokens )
14001421
function metaFuncs.concatTokens(tokens)
1401-
return concatTokens(tokens)
1422+
return _concatTokens(tokens)
14021423
end
14031424

14041425

@@ -1459,7 +1480,7 @@ local function _processFileOrString(params, isFile)
14591480
luaUnprocessed = rest
14601481
end
14611482

1462-
local tokensRaw = tokenize(luaUnprocessed, pathIn, params.backtickStrings, true)
1483+
local tokensRaw = _tokenize(luaUnprocessed, pathIn, true, params.backtickStrings, params.jitSyntax)
14631484
-- printTokens(tokensRaw)
14641485

14651486
-- Info variables.
@@ -1548,7 +1569,7 @@ local function _processFileOrString(params, isFile)
15481569
end
15491570
end
15501571

1551-
local toInsertTokens = tokenize(toInsertLua, toInsertName, params.backtickStrings, true)
1572+
local toInsertTokens = _tokenize(toInsertLua, toInsertName, true, params.backtickStrings, params.jitSyntax)
15521573
for i = #toInsertTokens, 1, -1 do
15531574
table.insert(tokenStack, toInsertTokens[i])
15541575
end
@@ -1589,7 +1610,7 @@ local function _processFileOrString(params, isFile)
15891610
local function flushTokensToProcess()
15901611
if not tokensToProcess[1] then return end
15911612

1592-
local lua = concatTokens(tokensToProcess, ln, params.addLineNumbers)
1613+
local lua = _concatTokens(tokensToProcess, ln, params.addLineNumbers)
15931614
local luaMeta
15941615

15951616
if isDebug then
@@ -1881,7 +1902,7 @@ local function _processFileOrString(params, isFile)
18811902
tokenIndex = tokenIndex+1
18821903
end
18831904

1884-
local metaBlock = concatTokens(tokensInBlock, nil, params.addLineNumbers)
1905+
local metaBlock = _concatTokens(tokensInBlock, nil, params.addLineNumbers)
18851906

18861907
if loadLuaString("return("..metaBlock..")") then
18871908
table.insert(metaParts, (doOutputLua and "__LUA((" or "__VAL(("))
@@ -2003,15 +2024,14 @@ local function _processFileOrString(params, isFile)
20032024
end
20042025

20052026
-- Check if the output is valid Lua.
2006-
--
2007-
-- @Incomplete: Maybe add an option to disable this? It might be useful if
2008-
-- e.g. Lua 5.1 is used to generate Lua 5.3 code (for whatever reason).
2009-
--
2010-
local luaToCheck = lua:gsub("^#![^\n]*", "")
2011-
local mainChunk, err = loadLuaString(luaToCheck, (isFile and params.pathMeta and "@" or "")..pathOut)
2012-
if not mainChunk then
2013-
local ln, _err = err:match'^.-:(%d+): (.*)'
2014-
errorOnLine(pathOut, (tonumber(ln) or 0), nil, "%s", (_err or err))
2027+
if params.validate ~= false then
2028+
local luaToCheck = lua:gsub("^#![^\n]*", "")
2029+
local mainChunk, err = loadLuaString(luaToCheck, (isFile and params.pathMeta and "@" or "")..pathOut)
2030+
2031+
if not mainChunk then
2032+
local ln, _err = err:match'^.-:(%d+): (.*)'
2033+
errorOnLine(pathOut, (tonumber(ln) or 0), nil, "%s", (_err or err))
2034+
end
20152035
end
20162036

20172037
-- :ProcessInfo
@@ -2131,6 +2151,8 @@ local lib = {
21312151
--
21322152
-- backtickStrings = boolean -- [Optional] Enable the backtick (`) to be used as string literal delimiters. Backtick strings don't interpret any escape sequences and can't contain other backticks. (Default: false)
21332153
-- canOutputNil = boolean -- [Optional] Allow !() and outputValue() to output nil. (Default: true)
2154+
-- jitSyntax = boolean -- [Optional] Allow LuaJIT-specific syntax. (Default: false)
2155+
-- validate = boolean -- [Optional] Validate output. (Default: true)
21342156
--
21352157
-- onInsert = function( name ) -- [Optional] Called for each @insert statement. It's expected to return a Lua string. By default 'name' is a path to a file to be inserted.
21362158
-- onBeforeMeta = function( ) -- [Optional] Called before the metaprogram runs.
@@ -2154,6 +2176,8 @@ local lib = {
21542176
--
21552177
-- backtickStrings = boolean -- [Optional] Enable the backtick (`) to be used as string literal delimiters. Backtick strings don't interpret any escape sequences and can't contain other backticks. (Default: false)
21562178
-- canOutputNil = boolean -- [Optional] Allow !() and outputValue() to output nil. (Default: true)
2179+
-- jitSyntax = boolean -- [Optional] Allow LuaJIT-specific syntax. (Default: false)
2180+
-- validate = boolean -- [Optional] Validate output. (Default: true)
21572181
--
21582182
-- onInsert = function( name ) -- [Optional] Called for each @insert statement. It's expected to return a Lua string. By default 'name' is a path to a file to be inserted.
21592183
-- onBeforeMeta = function( ) -- [Optional] Called before the metaprogram runs.

0 commit comments

Comments
 (0)