From a7403cf217012b9416e82f42c73f4ad887bdf04e Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Thu, 14 Dec 2023 20:17:58 +0100 Subject: [PATCH] the comment (#12) Improve comment handling by making sure that comment tokens are skipped consistently, postfixed at the "outer" level as much as possible and generally don't go missing if they get attached to the wrong node. * keep better track of end-of-token information for better empty line preservation * fix several cases of swallowed mid comments * fix several cases of unnecessarily re-indented comments * more regular line break after proc-with-body and friends --- nph.nimble | 5 +- src/astcmp.nim | 1 - src/phast.nim | 21 +- src/phastalgo.nim | 4 - src/phlexer.nim | 28 +- src/phlineinfos.nim | 7 +- src/phmsgs.nim | 8 - src/phoptions.nim | 13 +- src/phparser.nim | 532 ++++++------------ src/phrenderer.nim | 67 ++- tests/after/{empty.nim => a00empty.nim} | 0 ...pty.nim.nph.yaml => a00empty.nim.nph.yaml} | 0 tests/after/comments.nim | 83 ++- tests/after/comments.nim.nph.yaml | 396 +++++++++---- tests/after/exprs.nim | 3 + tests/after/exprs.nim.nph.yaml | 22 +- tests/after/procs.nim | 9 +- tests/after/types.nim | 1 - tests/before/{empty.nim => a00empty.nim} | 0 ...pty.nim.nph.yaml => a00empty.nim.nph.yaml} | 0 tests/before/comments.nim | 74 ++- tests/before/comments.nim.nph.yaml | 392 +++++++++---- tests/before/exprs.nim | 5 +- tests/before/exprs.nim.nph.yaml | 22 +- 24 files changed, 1009 insertions(+), 684 deletions(-) rename tests/after/{empty.nim => a00empty.nim} (100%) rename tests/after/{empty.nim.nph.yaml => a00empty.nim.nph.yaml} (100%) rename tests/before/{empty.nim => a00empty.nim} (100%) rename tests/before/{empty.nim.nph.yaml => a00empty.nim.nph.yaml} (100%) diff --git a/nph.nimble b/nph.nimble index 1edee5f..8c27391 100644 --- a/nph.nimble +++ b/nph.nimble @@ -29,11 +29,14 @@ task self, "Format nph itself": echo file exec "./nph " & file +import std/algorithm task f, "Format": build() cd "tests/before" - for file in listFiles("."): + # Sort tests so that 00_empty always is first, which makes it a convenient + # experimentation ground :) + for file in sorted(listFiles(".")): if file.len > 4 and file[^4..^1] == ".nim": echo file exec "../../nph " & file & " --outDir:../after --debug" diff --git a/src/astcmp.nim b/src/astcmp.nim index 0e90b8e..6e155f5 100644 --- a/src/astcmp.nim +++ b/src/astcmp.nim @@ -47,7 +47,6 @@ proc equivalent*(a, b: PNode): Outcome = # TODO the positive and negative ranges of integers are not the same - is this a problem? # what about more complex expressions? return Outcome(kind: Same) - # runnableExamples: static: ... turns into a staticStmt when broken up in # lines (!) if a.kind == nkCall and b.kind == nkStaticStmt and a.sons.len > 1 and diff --git a/src/phast.nim b/src/phast.nim index 5c666d2..45c764a 100644 --- a/src/phast.nim +++ b/src/phast.nim @@ -1040,8 +1040,7 @@ type ident*: PIdent else: sons*: TNodeSeq - when defined(nimsuggest): - endInfo*: TLineInfo + endInfo*: TLineInfo prefix*: seq[Token] # comments leading up to this node mid*: seq[Token] # comments in the middle of the node postfix*: seq[Token] # comments after the node @@ -1156,6 +1155,7 @@ type # for modules, it's a placeholder for compiler # generated code that will be appended to the # module after the sem pass (see appendToModule) + options*: TOptions position*: int # used for many different things: @@ -1168,14 +1168,17 @@ type # for modules, an unique index corresponding # to the module's fileIdx # for variables a slot index for the evaluator + offset*: int32 # offset of record field disamb*: int32 # disambiguation number; the basic idea is that # `___` + loc*: TLoc annex*: PLib # additional fields (seldom used, so we use a # reference to another object to save space) + when hasFFI: cname*: string # resolved C declaration name in importc decl, e.g.: @@ -1186,6 +1189,7 @@ type # it won't cause problems # for skModule the string literal to output for # deprecated modules. + when defined(nimsuggest): allUsages*: seq[TLineInfo] @@ -1219,19 +1223,23 @@ type # formal param list # for concepts, the concept body # else: unused + owner*: PSym # the 'owner' of the type sym*: PSym # types have the sym associated with them # it is used for converting types to strings + size*: BiggestInt # the size of the type in bytes # -1 means that the size is unkwown + align*: int16 # the type's alignment requirements paddingAtEnd*: int16 # loc*: TLoc typeInst*: PType # for generic instantiations the tyGenericInst that led to this # type. + uniqueId*: ItemId # due to a design mistake, we need to keep the real ID here as it # is required by the --incremental:on mode. @@ -1560,6 +1568,7 @@ proc getDeclPragma*(n: PNode): PNode = Empty Ident "int" ]# + if n[0].kind == nkPragmaExpr: result = n[0][1] else: @@ -1945,7 +1954,6 @@ proc assignType*(dest, src: PType) = dest.n = src.n dest.size = src.size dest.align = src.align - # this fixes 'type TLock = TSysLock': if src.sym != nil: if dest.sym != nil: @@ -1973,7 +1981,6 @@ proc exactReplica*(t: PType): PType = proc copySym*(s: PSym; idgen: IdGenerator): PSym = result = newSym(s.kind, s.name, idgen, s.owner, s.info, s.options) - #result.ast = nil # BUGFIX; was: s.ast which made problems result.typ = s.typ result.flags = s.flags @@ -1992,10 +1999,8 @@ proc createModuleAlias*( s: PSym; idgen: IdGenerator; newIdent: PIdent; info: TLineInfo; options: TOptions ): PSym = result = newSym(s.kind, newIdent, idgen, s.owner, info, options) - # keep ID! result.ast = s.ast - #result.id = s.id # XXX figure out what to do with the ID. result.flags = s.flags result.options = s.options @@ -2021,7 +2026,6 @@ proc newIdTable*(): TIdTable = proc resetIdTable*(x: var TIdTable) = x.counter = 0 - # clear and set to old initial size: setLen(x.data, 0) setLen(x.data, StartSize) @@ -2153,7 +2157,6 @@ template transitionNodeKindCommon(k: TNodeKind) = mid: obj.mid, postfix: obj.postfix, ) - # n.comment = obj.comment # shouldn't be needed, the address doesnt' change when defined(useNodeIds): n.id = obj.id @@ -2487,6 +2490,7 @@ proc toObjectFromRefPtrGeneric*(typ: PType): PType = A3 = ref object f2: int A4 = object f3: int ]# + result = typ while true: case result.kind @@ -2571,7 +2575,6 @@ proc newProcType*(info: TLineInfo; id: ItemId; owner: PSym): PType = result.n = newNodeI(nkFormalParams, info) rawAddSon(result, nil) # return type - # result.n[0] used to be `nkType`, but now it's `nkEffectList` because # the effects are now stored in there too ... this is a bit hacky, but as # usual we desperately try to save memory: diff --git a/src/phastalgo.nim b/src/phastalgo.nim index 4dc0a4a..e349951 100644 --- a/src/phastalgo.nim +++ b/src/phastalgo.nim @@ -274,13 +274,11 @@ proc sameIgnoreBacktickGensymInfo(a, b: string): bool = if aa != bb: return false - # the characters are identical: if i >= alen: # both cursors at the end: if j >= b.len: return true - # not yet at the end of 'b': return false elif j >= b.len: @@ -845,7 +843,6 @@ proc debug(n: PNode; conf: ConfigRef) = var this: DebugPrinter this.visited = initTable[pointer, int]() - #this.renderSymType = true this.useColor = not defined(windows) @@ -994,7 +991,6 @@ proc strTableInclReportConflict*( var it = t.data[h] if it == nil: break - # Semantic checking can happen multiple times thanks to templates # and overloading: (var x=@[]; x).mapIt(it). # So it is possible the very same sym is added multiple diff --git a/src/phlexer.nim b/src/phlexer.nim index baf5dce..de57f93 100644 --- a/src/phlexer.nim +++ b/src/phlexer.nim @@ -43,8 +43,8 @@ const '@', '~', ':' } UnaryMinusWhitelist = {' ', '\t', '\n', '\r', ',', ';', '(', '[', '{'} - # don't forget to update the 'highlite' module if these charsets should change + type TokType* = enum tkInvalid = "tkInvalid" @@ -192,16 +192,20 @@ type base*: NumericalBase # the numerical base; only valid for int # or float literals + spacing*: set[TokenSpacing] # spaces around token indent*: int # the indentation; != -1 if the token has been # preceded with indentation + ident*: PIdent # the parsed identifier iNumber*: BiggestInt # the parsed integer literal fNumber*: BiggestFloat # the parsed floating point literal literal*: string # the parsed (string) literal; and # documentation comments are here too + + prevLine*: uint16 # line at which the previous token ended line*, col*: int offsetA*, offsetB*: int # used for pretty printing so that literals @@ -214,9 +218,12 @@ type # if > 0 an indentation has already been read # this is needed because scanning comments # needs so much look-ahead + currLineIndent*: int errorHandler*: ErrorHandler cache*: IdentCache + tokenEnd*: TLineInfo + previousTokenEnd*: TLineInfo config*: ConfigRef printTokens: bool @@ -232,6 +239,7 @@ template ones(n): untyped = ((1 shl n) - 1) # for utf-8 conversion + proc isNimIdentifier*(s: string): bool = let sLen = s.len if sLen > 0 and s[0] in SymStartChars: @@ -413,7 +421,6 @@ proc getNumber(L: var Lexer; result: var Token) = L.bufpos = startpos # Use L.bufpos as pos because of matchChars matchChars(L, t, literalishChars) - # We must verify +/- specifically so that we're not past the literal if L.buf[L.bufpos] in {'+', '-'} and L.buf[L.bufpos - 1] in {'e', 'E'}: t.literal.add(L.buf[L.bufpos]) @@ -522,7 +529,6 @@ proc getNumber(L: var Lexer; result: var Token) = discard matchUnderscoreChars(L, result, {'0' .. '9'}) let endpos = L.bufpos - # Second stage, find out if there's a datatype suffix and handle it var postPos = endpos if L.buf[postPos] in {'\'', 'f', 'F', 'd', 'D', 'i', 'I', 'u', 'U'}: @@ -581,7 +587,6 @@ proc getNumber(L: var Lexer; result: var Token) = lexMessageLitNum(L, "invalid number suffix: '$1'", errPos) else: lexMessageLitNum(L, "invalid number suffix: '$1'", errPos) - # Is there still a literalish char awaiting? Then it's an error! if L.buf[postPos] in literalishChars or (L.buf[postPos] == '.' and L.buf[postPos + 1] in {'0' .. '9'}): @@ -724,7 +729,6 @@ proc getNumber(L: var Lexer; result: var Token) = if outOfRange: lexMessageLitNum(L, "number out of range: '$1'", startpos) - # Promote int literal to int64? Not always necessary, but more consistent if result.tokType == tkIntLit: if result.iNumber > high(int32) or result.iNumber < low(int32): @@ -762,13 +766,11 @@ proc handleHexChar(L: var Lexer; xi: var int; position: range[0 .. 4]) = of '"', '\'': if position <= 1: invalid() - # do not progress the bufpos here. if position == 0: inc(L.bufpos) else: invalid() - # Need to progress for `nim check` inc(L.bufpos) @@ -780,7 +782,6 @@ proc handleDecChars(L: var Lexer; xi: var int) = proc addUnicodeCodePoint(s: var string; i: int) = let i = cast[uint](i) - # inlined toUTF-8 to avoid unicode and strutils dependencies. let pos = s.len if i <= 127: @@ -945,7 +946,6 @@ proc getString(L: var Lexer; tok: var Token; mode: StringMode) = tok.tokType = tkTripleStrLit # long string literal: inc(pos, 2) # skip "" - # skip leading newline: if L.buf[pos] in {' ', '\t'}: var newpos = pos + 1 @@ -1063,7 +1063,6 @@ proc getCharacter(L: var Lexer; tok: var Token) = tokenEndIgnore(tok, L.bufpos - 1) const UnicodeOperatorStartChars = {'\226', '\194', '\195'} - # the allowed unicode characters ("∙ ∘ × ★ ⊗ ⊘ ⊙ ⊛ ⊠ ⊡ ∩ ∧ ⊓ ± ⊕ ⊖ ⊞ ⊟ ∪ ∨ ⊔") # all start with one of these. @@ -1221,7 +1220,6 @@ proc getOperator(L: var Lexer; tok: var Token) = endOperator(L, tok, pos, h) tokenEnd(tok, pos - 1) - # advance pos but don't store it in L.bufpos so the next token (which might # be an operator too) gets the preceding spaces: tok.spacing = tok.spacing - {tsTrailing, tsEof} @@ -1246,7 +1244,6 @@ proc getPrecedence*(tok: Token): int = case tok.tokType of tkOpr: let relevantChar = tok.ident.s[0] - # arrow like? if tok.ident.s.len > 1 and tok.ident.s[^1] == '>' and tok.ident.s[^2] in {'-', '~', '='}: @@ -1426,10 +1423,13 @@ proc skip(L: var Lexer; tok: var Token) = proc rawGetTok*(L: var Lexer; tok: var Token) = template atTokenEnd() {.dirty.} = - discard + L.previousTokenEnd.line = L.tokenEnd.line + L.previousTokenEnd.col = L.tokenEnd.col + L.tokenEnd.line = tok.line.uint16 + L.tokenEnd.col = getColNumber(L, L.bufpos).int16 - # TODO nimsuggest leftover reset(tok) + tok.prevLine = L.tokenEnd.line tok.indent = -1 diff --git a/src/phlineinfos.nim b/src/phlineinfos.nim index cb3e01b..9247bad 100644 --- a/src/phlineinfos.nim +++ b/src/phlineinfos.nim @@ -14,7 +14,6 @@ import "$nim"/compiler/[ropes, pathutils], std/[hashes, tables] const explanationsBaseUrl* = "https://nim-lang.github.io/Nim" - # was: "https://nim-lang.org/docs" but we're now usually showing devel docs # instead of latest release docs. @@ -333,18 +332,22 @@ type quotedName*: Rope # cached quoted short name for codegen # purposes + quotedFullName*: Rope # cached quoted full name for codegen # purposes + lines*: seq[string] # the source code of the module # used for better error messages and # embedding the original source in the # generated code + dirtyFile*: AbsoluteFile # the file that is actually read into memory # and parsed; usually "" but is used # for 'nimsuggest' + hash*: string # the checksum of the file dirty*: bool # for 'nimpretty' like tooling fullContent*: string @@ -368,7 +371,6 @@ type TErrorOutputs* = set[TErrorOutput] ERecoverableError* = object of ValueError - ESuggestDone* = object of ValueError proc `==`*(a, b: FileIndex): bool {.borrow.} @@ -399,6 +401,7 @@ type MsgConfig* = object ## does not need to be stored in the incremental cache trackPosAttached*: bool ## whether the tracking position was attached to ## some close token. + errorOutputs*: TErrorOutputs msgContext*: seq[tuple[info: TLineInfo, detail: string]] lastError*: TLineInfo diff --git a/src/phmsgs.nim b/src/phmsgs.nim index 1686cde..3a190b1 100644 --- a/src/phmsgs.nim +++ b/src/phmsgs.nim @@ -72,10 +72,8 @@ proc makeCString*(s: string): Rope = proc newFileInfo(fullPath: AbsoluteFile; projPath: RelativeFile): TFileInfo = result.fullPath = fullPath - #shallow(result.fullPath) result.projPath = projPath - #shallow(result.projPath) result.shortName = fullPath.extractFilename result.quotedName = result.shortName.makeCString @@ -128,7 +126,6 @@ proc fileInfoIdx*( canon = canonicalizePath(conf, filename) except OSError: canon = filename - # The compiler uses "filenames" such as `command line` or `stdin` # This flag indicates that we are working with such a path here pseudoPath = true @@ -428,7 +425,6 @@ proc msgWriteln*(conf: ConfigRef; s: string; flags: MsgFlags = {}) = write stderr, s writeLine(stderr, sep) - # On Windows stderr is fully-buffered when piped, regardless of C std. when defined(windows): flushFile(stderr) @@ -494,7 +490,6 @@ template styledMsgWriteln(args: varargs[typed]) = callStyledWriteLineStderr(args) else: callIgnoringStyle(writeLine, stderr, args) - # On Windows stderr is fully-buffered when piped, regardless of C std. when defined(windows): flushFile(stderr) @@ -615,7 +610,6 @@ proc sourceLine*(conf: ConfigRef; i: TLineInfo): string = return "" let num = numLines(conf, i.fileIndex) - # can happen if the error points to EOF: if i.line.int > num: return "" @@ -725,7 +719,6 @@ proc liMessage*( conf.toFileLineCol(info) & " " else: "" - # we could also show `conf.cmdInput` here for `projectIsCmd` var kindmsg = if kind.len > 0: @@ -941,7 +934,6 @@ proc genSuccessX*(conf: ConfigRef) = build.add "size" else: build.add debugModeHints - # pending https://github.com/timotheecour/Nim/issues/752, point to optimization.html if isDefined(conf, "danger"): flags.add " -d:danger" diff --git a/src/phoptions.nim b/src/phoptions.nim index d14821d..6c1d354 100644 --- a/src/phoptions.nim +++ b/src/phoptions.nim @@ -324,6 +324,7 @@ type name*: ptr string # not used beyond sorting purposes; name is also # part of 'qualifiedPath' + filePath*: string line*: int # Starts at 1 column*: int # Starts at 0 @@ -398,15 +399,18 @@ type arguments*: string ## the arguments to be passed to the program that ## should be run + ideCmd*: IdeCmd oldNewlines*: bool cCompiler*: TSystemCC # the used compiler modifiedyNotes*: TNoteKinds # notes that have been set/unset from either cmdline/configs + cmdlineNotes*: TNoteKinds # notes that have been set/unset from cmdline foreignPackageNotes*: TNoteKinds notes*: TNoteKinds # notes after resolving all logic(defaults, verbosity)/cmdline/configs + warningAsErrors*: TNoteKinds mainPackageNotes*: TNoteKinds mainPackageId*: int @@ -421,6 +425,7 @@ type ## We need to use a StringTableRef here as defined ## symbols are always guaranteed to be style ## insensitive. Otherwise hell would break lose. + packageCache*: StringTableRef nimblePaths*: seq[AbsoluteDir] searchPaths*: seq[AbsoluteDir] @@ -456,6 +461,7 @@ type externalToLink*: seq[string] # files to link in addition to the file # we compiled (*) + linkOptionsCmd*: string compileOptionsCmd*: seq[string] linkOptions*: string # (*) @@ -556,7 +562,7 @@ const { optObjCheck, optFieldCheck, optRangeCheck, optBoundsCheck, optOverflowCheck, optAssert, optWarns, optRefCheck, optHints, optStackTrace, optLineTrace, - # consider adding `optStackTraceMsgs` + # consider adding `optStackTraceMsgs` optTrMacros, optStyleCheck, optCursorInference } DefaultGlobalOptions* = {optThreadAnalysis, optExcessiveStackTrace, optJsBigInt64} @@ -669,7 +675,6 @@ proc newConfigRef*(): ConfigRef = initConfigRefCommon(result) setTargetFromSystem(result.target) - # enable colors by default on terminals if terminal.isatty(stderr): incl(result.globalOptions, optUseColors) @@ -857,11 +862,9 @@ proc setDefaultLibpath*(conf: ConfigRef) = var prefix = getPrefixDir(conf) conf.libpath = prefix / RelativeDir"lib" - # Special rule to support other tools (nimble) which import the compiler # modules and make use of them. let realNimPath = findExe("nim") - # Find out if $nim/../../lib/system.nim exists. let parentNimLibPath = realNimPath.parentDir.parentDir / "lib" if not fileExists(conf.libpath.string / "system.nim") and @@ -1118,7 +1121,6 @@ proc findProjectNimFile*(conf: ConfigRef; pkg: string): string = if ext == ".nimble": if nimblepkg.len == 0: nimblepkg = name - # Since nimble packages can have their source in a subfolder, # check the last folder we were in for a possible match. if dir != prev: @@ -1156,6 +1158,7 @@ proc canonicalImportAux*(conf: ConfigRef; file: AbsoluteFile): string = Shows the canonical module import, e.g.: system, std/tables, fusion/pointers, system/assertions, std/private/asciitables ]## + var ret = getRelativePathFromConfigPath(conf, file, isTitle = true) let dir = getNimbleFile(conf, $file).parentDir.AbsoluteDir diff --git a/src/phparser.nim b/src/phparser.nim index 008b30f..3dfb6b3 100644 --- a/src/phparser.nim +++ b/src/phparser.nim @@ -32,7 +32,7 @@ # that the lexer is trying to do comment layout / reflow / analysis in the # middle of lexing.. -import "$nim"/compiler/[llstream, idents, pathutils], std/[strutils] +import "$nim"/compiler/[llstream, idents, pathutils], std/[strutils, sequtils] import "."/[phast, phlexer, phlineinfos, phmsgs, phoptions] when defined(nimPreviewSlimSystem): import std/assertions @@ -53,6 +53,7 @@ type inSemiStmtList*: int emptyNode: PNode skipped*: seq[Token] + endLine: uint16 SymbolMode = enum smNormal @@ -70,76 +71,35 @@ type clMid clPostfix -proc isOperator*(tok: Token): bool - -proc getTok*(p: var Parser) - -proc parMessage*(p: Parser; msg: TMsgKind; arg: string = "") - -proc skipComment*(p: var Parser; node: PNode; commentLoc = clPostfix) - -proc newNodeP*(kind: TNodeKind; p: var Parser): PNode - -proc newIntNodeP*(kind: TNodeKind; intVal: BiggestInt; p: var Parser): PNode - -proc newFloatNodeP*(kind: TNodeKind; floatVal: BiggestFloat; p: var Parser): PNode - -proc newStrNodeP*(kind: TNodeKind; strVal: string; p: var Parser): PNode - -proc newIdentNodeP*(ident: PIdent; p: var Parser): PNode - -proc expectIdentOrKeyw*(p: Parser) - -proc expectIdent*(p: Parser) - -proc parLineInfo*(p: Parser): TLineInfo - -proc eat*(p: var Parser; tokType: TokType) - -proc skipInd*(p: var Parser) - -proc optPar*(p: var Parser) - -proc optInd*(p: var Parser; n: PNode; commentLoc = clPostfix) - -proc indAndComment*( - p: var Parser; n: PNode; commentLoc = clPostfix; maybeMissEquals = false -) - -proc setBaseFlags*(n: PNode; base: NumericalBase) - -proc parseSymbol*(p: var Parser; mode = smNormal): PNode - proc parseTry(p: var Parser; isExpr: bool): PNode - proc parseCase(p: var Parser): PNode - proc parseStmtPragma(p: var Parser): PNode - proc parsePragma(p: var Parser): PNode - proc postExprBlocks(p: var Parser; x: PNode): PNode - -proc parseExprStmt(p: var Parser): PNode - proc parseBlock(p: var Parser): PNode - proc primary(p: var Parser; mode: PrimaryMode): PNode - -proc simpleExprAux(p: var Parser; limit: int; mode: PrimaryMode): PNode # implementation - -template prettySection(body) = - body +proc simpleExprAux(p: var Parser; limit: int; mode: PrimaryMode): PNode proc getTok(p: var Parser) = ## Get the next token from the parser's lexer, and store it in the parser's ## `tok` member. + ## + ## Comments are placed in the "skipped" member so that logic looking for + ## specific node types can find what it's looking for. p.lineNumberPrevious = p.lex.lineNumber p.lineStartPrevious = p.lex.lineStart p.bufposPrevious = p.lex.bufpos rawGetTok(p.lex, p.tok) + var tmp = p.lex.lineNumber + while p.tok.tokType == tkComment: + p.tok.prevLine = uint16 tmp + p.skipped.add p.tok + + rawGetTok(p.lex, p.tok) + tmp = p.lex.lineNumber + p.hasProgress = true proc openParser*( @@ -187,17 +147,12 @@ proc parMessage(p: Parser; arg: string) = template withInd(p, body: untyped) = let oldInd = p.currInd - p.currInd = p.tok.indent body p.currInd = oldInd -template newlineWasSplitting(p: var Parser) = - discard - -# TODO nimpretty leftover template realInd(p): bool = p.tok.indent > p.currInd @@ -219,33 +174,6 @@ proc addComment(node: PNode; tok: Token; commentLoc: CommentLoc) = of clPostfix: node.postfix.add tok -proc rawSkipComment( - p: var Parser; node: PNode; commentLoc = clPostfix; ind = 0; maxInd = int.high -) = - while p.tok.tokType == tkComment and - (p.tok.indent == -1 or (p.tok.indent >= ind and p.tok.indent < maxInd)): - # debugEcho "rsk ", p.tok.indent, " ", ind, " ", maxInd, " ", $p.tok - if node != nil: - addComment(node, p.tok, commentLoc) - else: - parMessage(p, errInternal, "skipComment") - - getTok(p) - -proc skipComment(p: var Parser; node: PNode; commentLoc = clPostfix) = - rawSkipComment(p, node, commentLoc) - -proc indComment(p: var Parser; node: PNode; commentLoc = clPostfix) = - # Add all comments at the current indent level or greater to node - this - # is useful to catch comments in statement lists etc - rawSkipComment(p, node, commentLoc, p.currInd) - -proc commentLookahead(p: var Parser; maxInd = int.high): seq[Token] = - while p.tok.tokType == tkComment and p.tok.indent < maxInd: - result.add p.tok - - getTok(p) - const errInvalidIndentation = "invalid indentation $1" errIdentifierExpected = "identifier expected, but got '$1'" @@ -261,8 +189,9 @@ proc optPar(p: var Parser) = if p.tok.indent < p.currInd: parMessage(p, errInvalidIndentation % "optPar") -proc optInd(p: var Parser; n: PNode; commentLoc = clPostfix) = - skipComment(p, n, commentLoc) +proc optInd(p: var Parser; n: PNode) = + # see getTok for comment lookahead handling + # skipComment(p, n, commentLoc) skipInd(p) proc getTokNoInd(p: var Parser) = @@ -270,10 +199,6 @@ proc getTokNoInd(p: var Parser) = if p.tok.indent >= 0: parMessage(p, errInvalidIndentation % "getTokNoInd") -proc expectIdentOrKeyw(p: Parser) = - if p.tok.tokType != tkSymbol and not isKeyword(p.tok.tokType): - lexMessage(p.lex, errGenerated, errIdentifierExpected % prettyTok(p.tok)) - proc expectIdent(p: Parser) = if p.tok.tokType != tkSymbol: lexMessage(p.lex, errGenerated, errIdentifierExpected % prettyTok(p.tok)) @@ -295,6 +220,26 @@ proc parLineInfo(p: Parser): TLineInfo = ## Retrieve the line information associated with the parser's current state. result = getLineInfo(p.lex, p.tok) +template setEndInfo(node: PNode) = + # End info is used to compute original line breaks + # TODO this endinfo is not quite correct - we only need to track this for + # for statement lists, which is the only place where we retain whitespace + node.endInfo = + TLineInfo( + fileIndex: p.lex.fileIdx, + line: uint16 p.lineNumberPrevious, + col: p.lex.previousTokenEnd.col, + ) + +proc drainSkipped(p: var Parser; indent: int): seq[Token] = + if p.skipped.len > 0: + for c in p.skipped: + if c.indent == -1 or c.indent > indent: + result.add(c) + p.lineNumberPrevious = int c.prevLine + + p.skipped.delete(0 .. result.high) + proc splitComments( comments: openArray[Token]; a: PNode; ind: int; commentLoc: CommentLoc ): seq[Token] = @@ -304,16 +249,26 @@ proc splitComments( else: result.add(c) -proc splitLookahead(p: var Parser; n: PNode; commentLoc: CommentLoc) = - let comments = p.skipped & commentLookahead(p) +proc splitLookahead(p: var Parser; n: PNode; indent: int; commentLoc: CommentLoc) = + for c in p.drainSkipped(indent): + n.addComment(c, commentLoc) - p.skipped = splitComments(comments, n, p.tok.indent, commentLoc) +proc splitLookahead(p: var Parser; n: PNode; commentLoc: CommentLoc) = + # Split comments such that anything at greater indent than the next + # non-comment token is added to `n` at `commentLoc` + splitLookahead(p, n, p.tok.indent, commentLoc) proc addSkipped(p: var Parser; n: PNode) = + # Add skipped comments at current or greater indent level to n as comment + # nodes - it is best used after moving greater-indent level nodes to a postfix + # to avoid them moving around + var tmp = move(p.skipped) var i = 0 - while i < tmp.len and tmp[i].indent >= p.currInd: + while i < tmp.len and (tmp[i].indent < 0 or tmp[i].indent >= p.currInd): let c = newNodeI(nkCommentStmt, getLineInfo(p.lex, tmp[i])) + p.lineNumberPrevious = int tmp[i].prevLine + setEndInfo(c) c.comment = tmp[i].literal n.add c i += 1 @@ -323,7 +278,7 @@ proc addSkipped(p: var Parser; n: PNode) = proc indAndComment( p: var Parser; n: PNode; commentLoc = clPostfix; maybeMissEquals = false ) = - indComment(p, n, commentLoc) + # indComment(p, n, commentLoc) if p.tok.indent > p.currInd: if maybeMissEquals: let col = p.bufposPrevious - p.lineStartPrevious @@ -337,9 +292,10 @@ proc indAndComment( #raiseAssert $p.tok, " ", p.parLineInfo() parMessage(p, errInvalidIndentation % "indAndComment") -proc newNodeP(kind: TNodeKind; p: var Parser): PNode = +proc newNodeP(kind: TNodeKind; p: var Parser; withPrefix = true): PNode = result = newNodeI(kind, parLineInfo(p)) - result.prefix = move(p.skipped) + if withPrefix: + result.prefix = move(p.skipped) proc newIntNodeP(kind: TNodeKind; intVal: BiggestInt; p: var Parser): PNode = result = newNodeP(kind, p) @@ -358,13 +314,9 @@ proc newIdentNodeP(ident: PIdent; p: var Parser): PNode = result.ident = ident proc parseExpr(p: var Parser): PNode - proc parseStmt(p: var Parser): PNode - proc parseTypeDesc(p: var Parser; fullExpr = false): PNode - proc parseTypeDefValue(p: var Parser): PNode - proc parseParamList(p: var Parser; retColon = true): PNode proc isSigilLike(tok: Token): bool {.inline.} = @@ -442,8 +394,7 @@ proc parseColComStmt(p: var Parser; n: PNode; commentLoc = clPostfix): PNode = const tkBuiltInMagics = {tkType, tkStatic, tkAddr} template setEndInfo() = - # TODO nimsuggest leftover - discard + setEndInfo(result) proc parseSymbol(p: var Parser; mode = smNormal): PNode = #| symbol = '`' (KEYW|IDENT|literal|(operator|'('|')'|'['|']'|'{'|'}'|'=')+)+ '`' @@ -472,7 +423,6 @@ proc parseSymbol(p: var Parser; mode = smNormal): PNode = result = newNodeP(nkAccQuoted, p) getTok(p) - # progress guaranteed while true: case p.tok.tokType @@ -504,7 +454,6 @@ proc parseSymbol(p: var Parser; mode = smNormal): PNode = eat(p, tkAccent) else: parMessage(p, errIdentifierExpected, p.tok) - # BUGFIX: We must consume a token here to prevent endless loops! # But: this really sucks for idetools and keywords, so we don't do it # if it is a keyword: @@ -518,9 +467,7 @@ proc equals(p: var Parser; a: PNode): PNode = result = newNodeP(nkExprEqExpr, p) getTok(p) - - p.skipped.add commentLookahead(p) - + #optInd(p, result) result.add(a) result.add(parseExpr(p)) splitLookahead(p, result, clPostfix) @@ -532,11 +479,10 @@ proc colonOrEquals(p: var Parser; a: PNode): PNode = result = newNodeP(nkExprColonExpr, p) getTok(p) - splitLookahead(p, result, clPostfix) # TODO mid? + #optInd(p, result) result.add(a) result.add(parseExpr(p)) - splitLookahead(p, result, clPostfix) else: result = equals(p, a) @@ -561,11 +507,7 @@ proc exprEqExpr(p: var Parser): PNode = proc exprList(p: var Parser; endTok: TokType; result: PNode) = #| exprList = expr ^+ comma getTok(p) - - p.skipped.add commentLookahead(p) - - skipInd(p) - + optInd(p, result) # progress guaranteed var a = parseExpr(p) @@ -576,7 +518,7 @@ proc exprList(p: var Parser; endTok: TokType; result: PNode) = getTok(p) splitLookahead(p, a, clPostfix) - skipInd(p) + optInd(p, a) var a = parseExpr(p) @@ -586,10 +528,7 @@ proc optionalExprList(p: var Parser; endTok: TokType; result: PNode) = #| optionalExprList = expr ^* comma getTok(p) - p.skipped.add commentLookahead(p) - - skipInd(p) - + optInd(p, result) # progress guaranteed while (p.tok.tokType != endTok) and (p.tok.tokType != tkEof): var a = parseExpr(p) @@ -600,14 +539,13 @@ proc optionalExprList(p: var Parser; endTok: TokType; result: PNode) = getTok(p) splitLookahead(p, a, clPostfix) - skipInd(p) + optInd(p, a) proc exprColonEqExprListAux(p: var Parser; endTok: TokType; result: PNode) = assert(endTok in {tkCurlyRi, tkCurlyDotRi, tkBracketRi, tkParRi}) getTok(p) splitLookahead(p, result, clMid) optPar(p) - # progress guaranteed while p.tok.tokType != endTok and p.tok.tokType != tkEof: var a = exprColonEqExpr(p) @@ -636,19 +574,16 @@ proc dotExpr(p: var Parser; a: PNode): PNode = getTok(p) - result = newNodeP(nkDotExpr, p) + result = newNodeP(nkDotExpr, p, withPrefix = false) result.info = info - splitLookahead(p, result, clMid) - skipInd(p) + optInd(p, result) result.add(a) result.add(parseSymbol(p, smAfterDot)) - # TODO non-line-ending comments? # splitLookahead(p, result[^1], clPostfix) if p.tok.tokType == tkBracketLeColon and tsLeading notin p.tok.spacing: var x = newNodeP(nkBracketExpr, p) - # rewrite 'x.y[:z]()' to 'y[z](x)' x.add result[1] @@ -670,7 +605,7 @@ proc dotLikeExpr(p: var Parser; a: PNode): PNode = result = newNodeI(nkInfix, info) splitLookahead(p, result, clMid) - skipInd(p) + optInd(p, result) var opNode = newIdentNodeP(p.tok.ident, p) @@ -678,7 +613,6 @@ proc dotLikeExpr(p: var Parser; a: PNode): PNode = result.add(opNode) result.add(a) result.add(parseSymbol(p, smAfterDot)) - splitLookahead(p, result[^1], clPostfix) proc qualifiedIdent(p: var Parser): PNode = #| qualifiedIdent = symbol ('.' optInd symbolOrKeyword)? @@ -693,8 +627,7 @@ proc setOrTableConstr(p: var Parser): PNode = result = newNodeP(nkCurly, p) getTok(p) # skip '{' - p.skipped.add commentLookahead(p) - skipInd(p) + optInd(p, result) if p.tok.tokType == tkColon: getTok(p) # skip ':' result.transitionSonsKind(nkTableConstr) @@ -710,7 +643,6 @@ proc setOrTableConstr(p: var Parser): PNode = break getTok(p) - skipComment(p, a) optPar(p) eat(p, tkCurlyRi) # skip '}' @@ -723,22 +655,19 @@ proc parseCast(p: var Parser): PNode = getTok(p) if p.tok.tokType == tkBracketLe: getTok(p) - p.skipped.add commentLookahead(p) - skipInd(p) + optInd(p, result) result.add(parseTypeDesc(p)) optPar(p) eat(p, tkBracketRi) eat(p, tkParLe) - p.skipped.add commentLookahead(p) - skipInd(p) + optInd(p, result) result.add(parseExpr(p)) else: result.add p.emptyNode eat(p, tkParLe) - p.skipped.add commentLookahead(p) + optInd(p, result) - skipInd(p) result.add(exprColonEqExpr(p)) optPar(p) @@ -806,9 +735,7 @@ proc semiStmtList(p: var Parser; result: PNode) = let a = complexOrSimpleStmt(p) if a.kind == nkEmpty: - debugEcho "2" - - parMessage(p, errExprExpected, p.tok) + parMessage(p, errExprExpected & " (semiStmtList)", p.tok) getTok(p) else: result.add a @@ -835,10 +762,8 @@ proc parsePar(p: var Parser): PNode = result = newNodeP(nkPar, p) getTok(p) + optInd(p, result) - p.skipped.add commentLookahead(p) - - skipInd(p) if p.tok.tokType in { tkDiscard, tkInclude, tkIf, tkWhile, tkCase, tkTry, tkDefer, tkFinally, @@ -850,8 +775,7 @@ proc parsePar(p: var Parser): PNode = elif p.tok.tokType == tkSemiColon: # '(;' enforces 'stmt' context: getTok(p) - p.skipped.add commentLookahead(p) - skipInd(p) + optInd(p, result) semiStmtList(p, result) elif p.tok.tokType == tkCurlyDotLe: result.add(parseStmtPragma(p)) @@ -867,8 +791,7 @@ proc parsePar(p: var Parser): PNode = let asgn = newNodeP(nkAsgn, p) getTok(p) - p.skipped.add commentLookahead(p) - skipInd(p) + optInd(p, result) let b = parseExpr(p) @@ -890,11 +813,8 @@ proc parsePar(p: var Parser): PNode = result.add(a) if p.tok.tokType == tkComma: getTok(p) - skipComment(p, a) - # (1,) produces a tuple expression: result.transitionSonsKind(nkTupleConstr) - # progress guaranteed while p.tok.tokType != tkParRi and p.tok.tokType != tkEof: var a = exprColonEqExpr(p) @@ -904,7 +824,6 @@ proc parsePar(p: var Parser): PNode = break getTok(p) - skipComment(p, a) optPar(p) eat(p, tkParRi) @@ -1050,9 +969,7 @@ proc identOrLiteral(p: var Parser; mode: PrimaryMode): PNode = of tkCast: result = parseCast(p) else: - debugEcho "1" - - parMessage(p, errExprExpected, p.tok) + parMessage(p, errExprExpected & " (identOrLiteral)", p.tok) getTok(p) # we must consume a token here to prevent endless loops! result = p.emptyNode @@ -1063,7 +980,6 @@ proc namedParams(p: var Parser; callee: PNode; kind: TNodeKind; endTok: TokType) result = newNodeP(kind, p) result.add(a) - # progress guaranteed exprColonEqExprListAux(p, endTok, result) @@ -1084,16 +1000,15 @@ proc commandExpr(p: var Parser; r: PNode; mode: PrimaryMode): PNode = result = r else: result = newNodeP(nkCommand, p) - + let baseIndent = p.currInd result.add(r) var isFirstParam = true - # progress NOT guaranteed p.hasProgress = false result.add commandParam(p, isFirstParam, mode) - splitLookahead(p, result[^1], clPostfix) + splitLookahead(p, result[^1], baseIndent, clPostfix) proc isDotLike(tok: Token): bool = result = @@ -1108,7 +1023,6 @@ proc primarySuffix(p: var Parser; r: PNode; baseIndent: int; mode: PrimaryMode): #| | '{' optInd exprColonEqExprList optPar '}' # XXX strong spaces need to be reflected above result = r - # progress guaranteed while p.tok.indent < 0 or (p.tok.tokType == tkDot and p.tok.indent >= baseIndent): case p.tok.tokType @@ -1120,19 +1034,16 @@ proc primarySuffix(p: var Parser; r: PNode; baseIndent: int; mode: PrimaryMode): break result = namedParams(p, result, nkCall, tkParRi) - # After a parLe, it's tricky to inject doc comments so we'll move them to # the next token - # TODO investigate non-doc-comments - p.skipped.add commentLookahead(p) + # TODO investigate non-doc-comments and empty par if result.len > 1 and result[1].kind == nkExprColonExpr: result.transitionSonsKind(nkObjConstr) of tkDot: # progress guaranteed result = dotExpr(p, result) result = parseGStrLit(p, result) - - splitLookahead(p, result, clPostfix) + splitLookahead(p, result, baseIndent, clPostfix) of tkBracketLe: # progress guaranteed if tsLeading in p.tok.spacing: @@ -1195,7 +1106,6 @@ proc parseOperators( p: var Parser; headNode: PNode; limit: int; mode: PrimaryMode ): PNode = result = headNode - # expand while operators have priorities higher than 'limit' var opPrec = getPrecedence(p.tok) @@ -1204,7 +1114,6 @@ proc parseOperators( pmTypeDesc else: mode - # the operator itself must not start on a new line: # progress guaranteed while opPrec >= limit and p.tok.indent < 0 and not isUnary(p.tok): @@ -1218,7 +1127,6 @@ proc parseOperators( getTok(p) splitLookahead(p, opNode, clPostfix) optPar(p) - # read sub-expression with higher priority: var b = simpleExprAux(p, opPrec + leftAssoc, modeB) @@ -1260,6 +1168,7 @@ proc parsePragma(p: var Parser): PNode = getTok(p) splitLookahead(p, result, clMid) + optInd(p, result) while p.tok.tokType notin {tkCurlyDotRi, tkCurlyRi, tkEof}: p.hasProgress = false @@ -1330,7 +1239,6 @@ proc parseIdentColonEquals(p: var Parser; flags: DeclaredIdentFlags): PNode = var a: PNode result = newNodeP(nkIdentDefs, p) - # progress guaranteed while true: case p.tok.tokType @@ -1346,33 +1254,34 @@ proc parseIdentColonEquals(p: var Parser; flags: DeclaredIdentFlags): PNode = break result.add(a) - splitLookahead(p, a, clPostfix) if p.tok.tokType != tkComma: break - + # If there is a `:` or `=`, comments should be split between the + # ident here and what follows - else, it's up to the caller to deal with + # them so that we don't attach postfixes too deep in the AST + p.skipped = splitComments(p.skipped, a, p.tok.indent, clPostfix) getTok(p) - splitLookahead(p, a, clPostfix) - skipInd(p) + # Skip ahead to the colon or equals + optInd(p, a) if p.tok.tokType == tkColon: getTok(p) splitLookahead(p, result, clMid) - skipInd(p) + optInd(p, result) result.add(parseTypeDesc(p, fullExpr = true)) else: - result.add(newNodeP(nkEmpty, p)) + result.add p.emptyNode if p.tok.tokType != tkEquals and withBothOptional notin flags: parMessage(p, "':' or '=' expected, but got '$1'", p.tok) if p.tok.tokType == tkEquals: getTok(p) splitLookahead(p, result, clMid) - skipInd(p) + optInd(p, result) result.add(parseExpr(p)) else: - result.add(newNodeP(nkEmpty, p)) + result.add p.emptyNode - splitLookahead(p, result, clPostfix) setEndInfo() proc parseTuple(p: var Parser; indentAllowed = false): PNode = @@ -1385,11 +1294,7 @@ proc parseTuple(p: var Parser; indentAllowed = false): PNode = getTok(p) if p.tok.tokType == tkBracketLe: getTok(p) - - p.skipped.add commentLookahead(p) - - skipInd(p) - + optInd(p, result) # progress guaranteed while p.tok.tokType in {tkSymbol, tkAccent}: var a = parseIdentColonEquals(p, {}) @@ -1409,7 +1314,6 @@ proc parseTuple(p: var Parser; indentAllowed = false): PNode = if realInd(p): withInd(p): splitLookahead(p, result, clMid) - # progress guaranteed while true: case p.tok.tokType @@ -1445,7 +1349,7 @@ proc parseParamList(p: var Parser; retColon = true): PNode = #| paramListColon = paramList? (':' optInd typeDesc)? var a: PNode - result = newNodeP(nkFormalParams, p) + result = newNodeP(nkFormalParams, p, withPrefix = false) result.add(p.emptyNode) # return type @@ -1453,8 +1357,7 @@ proc parseParamList(p: var Parser; retColon = true): PNode = if hasParLe: getTok(p) splitLookahead(p, result, clMid) - skipInd(p) - + optInd(p, result) # progress guaranteed while true: case p.tok.tokType @@ -1474,13 +1377,15 @@ proc parseParamList(p: var Parser; retColon = true): PNode = break result.add(a) - skipComment(p, a) + splitLookahead(p, a, clPostfix) if p.tok.tokType notin {tkComma, tkSemiColon}: break getTok(p) - skipComment(p, a) - + # Not ideal, but we'll attach the last comment in the par to the last + # parameter + if result.len > 0: + splitLookahead(p, result[^1], clPostfix) optPar(p) eat(p, tkParRi) @@ -1492,8 +1397,7 @@ proc parseParamList(p: var Parser; retColon = true): PNode = if hasRet and p.tok.indent < 0: getTok(p) - p.skipped.add commentLookahead(p) - skipInd(p) + optInd(p, result) result[0] = parseTypeDesc(p) elif not retColon and not hasParLe: @@ -1646,7 +1550,7 @@ proc parseSymbolList(p: var Parser; result: PNode) = getTok(p) splitLookahead(p, s, clPostfix) - skipInd(p) + optInd(p, s) setEndInfo() @@ -1657,8 +1561,7 @@ proc parseTypeDescKAux(p: var Parser; kind: TNodeKind; mode: PrimaryMode): PNode if p.tok.indent != -1 and p.tok.indent <= p.currInd: return - p.skipped.add commentLookahead(p) - skipInd(p) + optInd(p, result) let isTypedef = mode == pmTypeDef and p.tok.tokType in {tkObject, tkTuple} if not isOperator(p.tok) and isExprStart(p): @@ -1707,7 +1610,7 @@ proc parseFor(p: var Parser): PNode = while p.tok.tokType == tkComma: getTok(p) splitLookahead(p, a, clPostfix) - skipInd(p) + optInd(p, a) if p.tok.tokType == tkParLe: result.add(parseVarTuple(p)) @@ -1781,6 +1684,7 @@ proc primary(p: var Parser; mode: PrimaryMode): PNode = result.add(a) getTok(p) splitLookahead(p, a, clPostfix) + optInd(p, a) const identOrLiteralKinds = tkBuiltInMagics + @@ -1800,7 +1704,7 @@ proc primary(p: var Parser; mode: PrimaryMode): PNode = let baseInd = p.lex.currLineIndent result.add(identOrLiteral(p, mode)) - + splitLookahead(p, result[^1], clPostfix) result = primarySuffix(p, result, baseInd, mode) else: result.add(primary(p, pmNormal)) @@ -1826,7 +1730,7 @@ proc primary(p: var Parser; mode: PrimaryMode): PNode = getTok(p) splitLookahead(p, result, clPostfix) # TODO mid - skipInd(p) + optInd(p, result) result.add(primary(p, pmNormal)) of tkTuple, tkEnum, tkObject, tkConcept, tkVar, tkOut, tkRef, tkPtr, tkDistinct: result = parseTypeDesc(p) @@ -1834,6 +1738,9 @@ proc primary(p: var Parser; mode: PrimaryMode): PNode = let baseInd = p.lex.currLineIndent result = identOrLiteral(p, mode) + # Make sure that primarySuffix has the right token to work with, but don't + # consume comments just yet (so they can be attached to an appropriate + # outer node if there is no suffix) result = primarySuffix(p, result, baseInd, mode) proc binaryNot(p: var Parser; a: PNode): PNode = @@ -1841,9 +1748,7 @@ proc binaryNot(p: var Parser; a: PNode): PNode = let notOpr = newIdentNodeP(p.tok.ident, p) getTok(p) - p.skipped.add commentLookahead(p) - - skipInd(p) + optInd(p, notOpr) let b = primary(p, pmTypeDesc) @@ -1861,7 +1766,6 @@ proc parseTypeDesc(p: var Parser; fullExpr = false): PNode = #| ('not' primary)? #| typeDescExpr = (routineType / simpleExpr) ('not' primary)? #| typeDesc = rawTypeDesc / typeDescExpr - newlineWasSplitting(p) if fullExpr: result = simpleExpr(p, pmTypeDesc) else: @@ -1918,11 +1822,9 @@ proc parseTypeDefValue(p: var Parser): PNode = of tkDistinct: result = parseTypeDescKAux(p, nkDistinctTy, pmTypeDef) of tkEnum: - prettySection: - result = parseEnum(p) + result = parseEnum(p) of tkObject: - prettySection: - result = parseObject(p) + result = parseObject(p) of tkConcept: result = parseTypeClass(p) else: @@ -1934,7 +1836,7 @@ proc parseTypeDefValue(p: var Parser): PNode = getTok(p) assert result.len > 0 splitLookahead(p, result[^1], clPostfix) - skipInd(p) + optInd(p, result) result.add(commandParam(p, isFirstParam, pmTypeDef)) @@ -1983,10 +1885,9 @@ proc postExprBlocks(p: var Parser; x: PNode): PNode = getTok(p) splitLookahead(p, result, clMid) if not (p.tok.tokType in {tkOf, tkElif, tkElse, tkExcept, tkFinally} and sameInd(p)): - var stmtList = newNodeP(nkStmtList, p) + var stmtList = newNodeP(nkStmtList, p, withPrefix = false) stmtList.add parseStmt(p) - # to keep backwards compatibility (see tests/vm/tstringnil) if stmtList[0].kind == nkStmtList: stmtList = stmtList[0] @@ -2035,7 +1936,7 @@ proc postExprBlocks(p: var Parser; x: PNode): PNode = getTok(p) splitLookahead(p, nextBlock, clMid) - skipInd(p) + optInd(p, nextBlock) nextBlock.add parseExpr(p) of tkExcept: @@ -2070,14 +1971,13 @@ proc parseExprStmt(p: var Parser): PNode = #| / simplePrimary (exprEqExpr ^+ comma) postExprBlocks? #| / simpleExpr '=' optInd (expr postExprBlocks?) var a = simpleExpr(p, pmTrySimple) - p.skipped.add commentLookahead(p) if p.tok.tokType == tkEquals: result = newNodeP(nkAsgn, p) getTok(p) splitLookahead(p, result, clMid) - skipInd(p) + optInd(p, result) var b = parseExpr(p) @@ -2087,7 +1987,6 @@ proc parseExprStmt(p: var Parser): PNode = result.add(b) else: var isFirstParam = false - # if an expression is starting here, a simplePrimary was parsed and # this is the start of a command if p.tok.indent < 0 and isExprStart(p): @@ -2102,12 +2001,11 @@ proc parseExprStmt(p: var Parser): PNode = getTok(p) splitLookahead(p, result[^1], clPostfix) - skipInd(p) - + optInd(p, result) # This attaches any eol comments to the last argument instead of the # command which may or may not be a good idea, specially if those comments # are doc comments - splitLookahead(p, result[^1], clPostfix) + splitLookahead(p, result[^1], baseIndent, clPostfix) else: result = a @@ -2133,7 +2031,7 @@ proc parseImport(p: var Parser; kind: TNodeKind): PNode = splitLookahead(p, result, clMid) - skipInd(p) + optInd(p, result) var a = parseModuleName(p, kind) @@ -2145,6 +2043,7 @@ proc parseImport(p: var Parser; kind: TNodeKind): PNode = getTok(p) splitLookahead(p, a, clPostfix) + optInd(p, result) while true: # was: while p.tok.tokType notin {tkEof, tkSad, tkDed}: p.hasProgress = false @@ -2159,8 +2058,7 @@ proc parseImport(p: var Parser; kind: TNodeKind): PNode = getTok(p) splitLookahead(p, a, clPostfix) - skipInd(p) - + optInd(p, a) #expectNl(p) setEndInfo() @@ -2171,6 +2069,7 @@ proc parseIncludeStmt(p: var Parser): PNode = getTok(p) # skip `import` or `include` splitLookahead(p, result, clMid) + optInd(p, result) while true: # was: while p.tok.tokType notin {tkEof, tkSad, tkDed}: @@ -2187,8 +2086,7 @@ proc parseIncludeStmt(p: var Parser): PNode = getTok(p) splitLookahead(p, a, clPostfix) - skipInd(p) - + optInd(p, a) #expectNl(p) setEndInfo() @@ -2197,20 +2095,14 @@ proc parseFromStmt(p: var Parser): PNode = result = newNodeP(nkFromStmt, p) getTok(p) # skip `from` - - p.skipped.add commentLookahead(p) - - skipInd(p) + optInd(p, result) var a = parseModuleName(p, nkImportStmt) splitLookahead(p, a, clPrefix) # TODO 'mid' for from? result.add(a) #optInd(p, a); eat(p, tkImport) - - p.skipped.add commentLookahead(p) - - skipInd(p) + optInd(p, result) while true: # p.tok.tokType notin {tkEof, tkSad, tkDed}: p.hasProgress = false @@ -2225,8 +2117,7 @@ proc parseFromStmt(p: var Parser): PNode = getTok(p) splitLookahead(p, a, clPostfix) - skipInd(p) - + optInd(p, a) #expectNl(p) setEndInfo() @@ -2240,11 +2131,9 @@ proc parseReturnOrRaise(p: var Parser; kind: TNodeKind): PNode = result = newNodeP(kind, p) getTok(p) - # We don't split lookahead here because it might be that we end up with no # expression - in this case, we don't want the comments attached to # the discard midpoint - question is, what to do with them? - p.skipped.add commentLookahead(p) if p.tok.indent >= 0 and p.tok.indent <= p.currInd or not isExprStart(p): # NL terminates: result.add(p.emptyNode) @@ -2274,7 +2163,7 @@ proc parseIfOrWhen(p: var Parser; kind: TNodeKind): PNode = var branch = newNodeP(nkElifBranch, p) splitLookahead(p, branch, clMid) - skipInd(p) + optInd(p, branch) branch.add(parseExpr(p)) branch.add(parseColComStmt(p, branch, clMid)) result.add(branch) @@ -2304,7 +2193,7 @@ proc parseIfOrWhenExpr(p: var Parser; kind: TNodeKind): PNode = var branch = newNodeP(nkElifExpr, p) splitLookahead(p, branch, clMid) - skipInd(p) + optInd(p, branch) branch.add(parseExpr(p)) branch.add(parseColComStmt(p, branch, clMid)) result.add(branch) @@ -2327,7 +2216,7 @@ proc parseWhile(p: var Parser): PNode = getTok(p) splitLookahead(p, result, clMid) - skipInd(p) + optInd(p, result) result.add(parseExpr(p)) result.add(parseColComStmt(p, result, clMid)) setEndInfo() @@ -2373,27 +2262,18 @@ proc parseCase(p: var Parser): PNode = b = newNodeP(nkElifBranch, p) getTok(p) - - p.skipped.add commentLookahead(p) - - skipInd(p) + optInd(p, b) b.add(parseExpr(p)) - - p.skipped.add commentLookahead(p) of tkElse: b = newNodeP(nkElse, p) getTok(p) - - p.skipped.add commentLookahead(p) else: break b.add(parseColComStmt(p, b, clMid)) - - p.skipped.add commentLookahead(p) - result.add(b) + if b.kind == nkElse: break @@ -2499,7 +2379,6 @@ proc parseGenericParam(p: var Parser): PNode = var a: PNode result = newNodeP(nkIdentDefs, p) - # progress guaranteed while true: case p.tok.tokType @@ -2533,12 +2412,12 @@ proc parseGenericParam(p: var Parser): PNode = getTok(p) splitLookahead(p, a, clPostfix) - skipInd(p) + optInd(p, a) if p.tok.tokType == tkColon: getTok(p) splitLookahead(p, result, clPostfix) # TODO mid? - skipInd(p) + optInd(p, result) result.add(parseExpr(p)) else: result.add(p.emptyNode) @@ -2546,7 +2425,7 @@ proc parseGenericParam(p: var Parser): PNode = if p.tok.tokType == tkEquals: getTok(p) splitLookahead(p, result, clPostfix) # TODO mid? - skipInd(p) + optInd(p, result) result.add(parseExpr(p)) else: result.add(p.emptyNode) @@ -2559,11 +2438,7 @@ proc parseGenericParamList(p: var Parser): PNode = result = newNodeP(nkGenericParams, p) getTok(p) - - p.skipped.add commentLookahead(p) - - skipInd(p) - + optInd(p, result) # progress guaranteed while p.tok.tokType in {tkSymbol, tkAccent, tkIn, tkOut}: var a = parseGenericParam(p) @@ -2596,6 +2471,7 @@ proc parseRoutine(p: var Parser; kind: TNodeKind): PNode = getTok(p) splitLookahead(p, result, clMid) + optInd(p, result) if kind in {nkProcDef, nkLambda, nkIteratorDef, nkFuncDef} and p.tok.tokType notin {tkSymbol, tokKeywordLow .. tokKeywordHigh, tkAccent}: # no name; lambda or proc type @@ -2629,7 +2505,6 @@ proc parseRoutine(p: var Parser; kind: TNodeKind): PNode = result.add(p.parsePragma) else: result.add(p.emptyNode) - # empty exception tracking: result.add(p.emptyNode) @@ -2638,13 +2513,11 @@ proc parseRoutine(p: var Parser; kind: TNodeKind): PNode = getTok(p) splitLookahead(p, result, clMid) - # debugEcho 333, p.tok.tokType result.add(parseStmt(p)) else: result.add(p.emptyNode) - # TODO commentlookahead - indAndComment(p, result, maybeMissEquals = maybeMissEquals) + indAndComment(p, result, maybeMissEquals = maybeMissEquals) # no comments really let body = result[^1] if body.kind == nkStmtList and body.len > 0 and body[0].comment.len > 0 and @@ -2664,6 +2537,7 @@ proc parseCommentStmt(p: var Parser): PNode = #| commentStmt = COMMENT result = newNodeP(nkCommentStmt, p) result.comment = p.tok.literal + setEndInfo() getTok(p) @@ -2671,7 +2545,7 @@ proc parseSection( p: var Parser; kind: TNodeKind; defparser: proc(p: var Parser): PNode {.nimcall.} ): PNode = #| section(RULE) = COMMENT? RULE / (IND{>} (RULE / COMMENT)^+IND{=} DED) - result = newNodeP(kind, p) + result = newNodeP(kind, p, withPrefix = false) if kind != nkTypeSection: getTok(p) @@ -2710,29 +2584,32 @@ proc parseEnum(p: var Parser): PNode = getTok(p) result.add(p.emptyNode) - splitLookahead(p, result, clMid) + optInd(p, result) + splitLookahead(p, result, clMid) # progress guaranteed while true: + let symInd = + if p.tok.indent == -1: + p.currInd + else: + p.tok.indent + var a = parseSymbol(p) if a.kind == nkEmpty: return - var comments = commentLookahead(p) var symPragma = a var pragma: PNode if (p.tok.indent < 0 or p.tok.indent >= p.currInd) and p.tok.tokType == tkCurlyDotLe: pragma = optPragmas(p) - symPragma = newNodeP(nkPragmaExpr, p) + symPragma = newNodeP(nkPragmaExpr, p, withPrefix = false) symPragma.add(a) symPragma.add(pragma) - comments.add commentLookahead(p) - - let symInd = p.currInd if p.tok.indent >= 0 and p.tok.indent <= p.currInd: - p.skipped.add splitComments(comments, symPragma, symInd, clPostfix) + p.skipped = splitComments(p.skipped, symPragma, symInd, clPostfix) result.add(symPragma) @@ -2740,10 +2617,7 @@ proc parseEnum(p: var Parser): PNode = if p.tok.tokType == tkEquals and p.tok.indent < 0: getTok(p) - - comments.add commentLookahead(p) - - skipInd(p) + optInd(p, symPragma) var b = symPragma @@ -2752,14 +2626,10 @@ proc parseEnum(p: var Parser): PNode = symPragma.add(b) symPragma.add(parseExpr(p)) - comments.add commentLookahead(p) - if p.tok.tokType == tkComma and p.tok.indent < 0: getTok(p) - comments.add commentLookahead(p) - - p.skipped.add splitComments(comments, symPragma, symInd, clPostfix) + p.skipped = splitComments(p.skipped, symPragma, symInd, clPostfix) result.add(symPragma) if p.tok.indent >= 0 and p.tok.indent <= p.currInd or p.tok.tokType == tkEof: @@ -2777,7 +2647,6 @@ proc parseObjectWhen(p: var Parser): PNode = #| ('elif' expr colcom objectPart COMMENT?)* #| ('else' colcom objectPart COMMENT?)? result = newNodeP(nkRecWhen, p) - # progress guaranteed while sameInd(p): getTok(p) # skip `when`, `elif` @@ -2785,7 +2654,7 @@ proc parseObjectWhen(p: var Parser): PNode = var branch = newNodeP(nkElifBranch, p) splitLookahead(p, branch, clMid) - skipInd(p) + optInd(p, branch) branch.add(parseExpr(p)) eat(p, tkColon) splitLookahead(p, branch, clMid) @@ -2798,12 +2667,9 @@ proc parseObjectWhen(p: var Parser): PNode = var branch = newNodeP(nkElse, p) eat(p, tkElse) - - p.skipped.add commentLookahead(p) - eat(p, tkColon) splitLookahead(p, branch, clMid) - skipInd(p) + optInd(p, result) branch.add(parseObjectPart(p)) result.add(branch) @@ -2823,6 +2689,7 @@ proc parseObjectCase(p: var Parser): PNode = getTokNoInd(p) var a = parseIdentColonEquals(p, {withPragma}) + splitLookahead(p, result, clMid) result.add(a) if p.tok.tokType == tkColon: @@ -2835,7 +2702,6 @@ proc parseObjectCase(p: var Parser): PNode = if realInd(p): p.currInd = p.tok.indent wasIndented = true - # progress guaranteed while sameInd(p): var b: PNode @@ -2881,6 +2747,7 @@ proc parseObjectPart(p: var Parser): PNode = case p.tok.tokType of tkCase, tkWhen, tkSymbol, tkAccent, tkNil, tkDiscard: result.add(parseObjectPart(p)) + splitLookahead(p, result[^1], p.currInd, clPostfix) else: parMessage(p, errIdentifierExpected, p.tok) @@ -2910,6 +2777,7 @@ proc parseObject(p: var Parser): PNode = result = newNodeP(nkObjectTy, p) getTok(p) + result.add(p.emptyNode) # compatibility with old pragma node if p.tok.tokType == tkOf and p.tok.indent < 0: var a = newNodeP(nkOfInherit, p) @@ -2921,7 +2789,6 @@ proc parseObject(p: var Parser): PNode = result.add(p.emptyNode) splitLookahead(p, result, clMid) - # an initial IND{>} HAS to follow: if not realInd(p): result.add(p.emptyNode) @@ -2967,8 +2834,6 @@ proc parseTypeClass(p: var Parser): PNode = result = newNodeP(nkTypeClassTy, p) getTok(p) - if p.tok.tokType == tkComment: - skipComment(p, result) if p.tok.indent < 0: var args = newNodeP(nkArgList, p) @@ -2990,7 +2855,6 @@ proc parseTypeClass(p: var Parser): PNode = var a = newNodeP(nkOfInherit, p) getTok(p) - # progress guaranteed while true: a.add(parseTypeDesc(p)) @@ -3002,10 +2866,6 @@ proc parseTypeClass(p: var Parser): PNode = result.add(a) else: result.add(p.emptyNode) - - if p.tok.tokType == tkComment: - skipComment(p, result) - # an initial IND{>} HAS to follow: if not realInd(p): if result.isNewStyleConcept: @@ -3032,20 +2892,17 @@ proc parseTypeDef(p: var Parser): PNode = var pragma: PNode var genericParam: PNode var next = identifier - p.skipped.add commentLookahead(p) + if p.tok.tokType == tkBracketLe and p.validInd: p.skipped = splitComments(p.skipped, identifier, p.tok.indent, clPostfix) genericParam = parseGenericParamList(p) - p.skipped.add commentLookahead(p) next = genericParam else: genericParam = p.emptyNode - # pragma = optPragmas(p) if p.tok.tokType == tkCurlyDotLe and (p.tok.indent < 0 or realInd(p)): p.skipped = splitComments(p.skipped, next, p.tok.indent, clPostfix) pragma = parsePragma(p) - p.skipped.add commentLookahead(p) next = pragma else: pragma = p.emptyNode @@ -3062,14 +2919,10 @@ proc parseTypeDef(p: var Parser): PNode = result.info = parLineInfo(p) getTok(p) - - p.skipped.add commentLookahead(p) - - skipInd(p) + optInd(p, result) result.add(parseTypeDefValue(p)) - p.skipped.add commentLookahead(p) p.skipped = splitComments(p.skipped, result[^1], p.currInd, clPostfix) else: result.add(p.emptyNode) @@ -3082,9 +2935,7 @@ proc parseVarTuple(p: var Parser): PNode = result = newNodeP(nkVarTuple, p) getTok(p) # skip '(' - - p.skipped.add commentLookahead(p) - + optInd(p, result) # progress guaranteed while p.tok.tokType in {tkSymbol, tkAccent, tkParLe}: var a: PNode @@ -3115,7 +2966,7 @@ proc parseVariable(p: var Parser): PNode = eat(p, tkEquals) splitLookahead(p, result, clMid) - skipInd(p) + optInd(p, result) result.add(parseExpr(p)) splitLookahead(p, result[^1], clPostfix) else: @@ -3123,7 +2974,6 @@ proc parseVariable(p: var Parser): PNode = result[^1] = postExprBlocks(p, result[^1]) - splitLookahead(p, result[^1], clPostfix) setEndInfo() proc parseConstant(p: var Parser): PNode = @@ -3137,15 +2987,14 @@ proc parseConstant(p: var Parser): PNode = if p.tok.tokType == tkColon: getTok(p) splitLookahead(p, result, clMid) - skipInd(p) + optInd(p, result) result.add(parseTypeDesc(p)) else: result.add(p.emptyNode) eat(p, tkEquals) splitLookahead(p, result, clMid) - skipInd(p) - + optInd(p, result) #add(result, parseStmtListExpr(p)) result.add(parseExpr(p)) @@ -3159,9 +3008,7 @@ proc parseBind(p: var Parser; k: TNodeKind): PNode = result = newNodeP(k, p) getTok(p) - p.skipped.add commentLookahead(p) # TODO mid? - skipInd(p) - + optInd(p, result) # TODO mid? # progress guaranteed while true: var a = qualifiedIdent(p) @@ -3172,8 +3019,7 @@ proc parseBind(p: var Parser; k: TNodeKind): PNode = getTok(p) splitLookahead(p, a, clPostfix) - skipInd(p) - + optInd(p, a) #expectNl(p) setEndInfo() @@ -3299,14 +3145,11 @@ proc complexOrSimpleStmt(p: var Parser): PNode = else: result = parseSection(p, nkTypeSection, parseTypeDef) of tkConst: - prettySection: - result = parseSection(p, nkConstSection, parseConstant) + result = parseSection(p, nkConstSection, parseConstant) of tkLet: - prettySection: - result = parseSection(p, nkLetSection, parseVariable) + result = parseSection(p, nkLetSection, parseVariable) of tkVar: - prettySection: - result = parseSection(p, nkVarSection, parseVariable) + result = parseSection(p, nkVarSection, parseVariable) of tkWhen: result = parseIfOrWhen(p, nkWhenStmt) of tkBind: @@ -3322,14 +3165,13 @@ proc parseStmt(p: var Parser): PNode = #| stmt = (IND{>} complexOrSimpleStmt^+(IND{=} / ';') DED) #| / simpleStmt ^+ ';' if p.tok.indent > p.currInd: - # nimpretty support here - result = newNodeP(nkStmtList, p) + result = newNodeP(nkStmtList, p, withPrefix = false) + addSkipped(p, result) withInd(p): splitLookahead(p, result, clPrefix) while true: - if p.tok.indent == p.currInd or - p.tok.indent > p.currInd and p.tok.tokType == tkComment: + if p.tok.indent == p.currInd: discard elif p.tok.tokType == tkSemiColon: getTok(p) @@ -3338,8 +3180,6 @@ proc parseStmt(p: var Parser): PNode = else: break else: - # TODO this hack causes reindentation of the comment to the "current" - # level when the comment is indented more than the last statement if p.tok.indent > p.currInd and p.tok.tokType notin {tkDot, tkComment}: printTok(p.lex.config, p.tok) parMessage(p, errInvalidIndentation % "parseStmt") @@ -3356,20 +3196,18 @@ proc parseStmt(p: var Parser): PNode = p.hasProgress = false if p.tok.tokType in {tkElse, tkElif}: break - # Allow this too, see tests/parser/tifexprs let a = complexOrSimpleStmt(p) if a.kind == nkEmpty and not p.hasProgress: - debugEcho 3 - - parMessage(p, errExprExpected, p.tok) + parMessage(p, errExprExpected & " (parseStmt >)", p.tok) break else: result.add a - p.skipped.add commentLookahead(p) - p.skipped = splitComments(p.skipped, a, p.currInd, clPostfix) + splitLookahead(p, a, p.currInd, clPostfix) + addSkipped(p, result) + if not p.hasProgress and p.tok.tokType == tkEof: break else: @@ -3384,11 +3222,10 @@ proc parseStmt(p: var Parser): PNode = if p.inSemiStmtList > 0: result = simpleStmt(p) if result.kind == nkEmpty: - debugEcho 4 - - parMessage(p, errExprExpected, p.tok) + parMessage(p, errExprExpected & " (parseStmt semi)", p.tok) else: - result = newNodeP(nkStmtList, p) + result = newNodeP(nkStmtList, p, withPrefix = false) + addSkipped(p, result) while true: # if p.tok.indent >= 0: # parMessage(p, errInvalidIndentation % "parseStmt 2") @@ -3397,9 +3234,7 @@ proc parseStmt(p: var Parser): PNode = let a = simpleStmt(p) let err = not p.hasProgress if a.kind == nkEmpty: - debugEcho 5 - - parMessage(p, errExprExpected, p.tok) + parMessage(p, errExprExpected & " (parseStmt =)", p.tok) result.add(a) if p.tok.tokType != tkSemiColon: @@ -3413,18 +3248,17 @@ proc parseStmt(p: var Parser): PNode = proc parseAll(p: var Parser): PNode = ## Parses the rest of the input stream held by the parser into a PNode. - result = newNodeP(nkStmtList, p) + result = newNodeP(nkStmtList, p, withPrefix = false) + addSkipped(p, result) + while p.tok.tokType != tkEof: p.hasProgress = false - var a = complexOrSimpleStmt(p) + let a = complexOrSimpleStmt(p) if a.kind != nkEmpty and p.hasProgress: result.add(a) else: - debugEcho 6 - - parMessage(p, errExprExpected, p.tok) - + parMessage(p, errExprExpected & " (parseAll)", p.tok) # bugfix: consume a token here to prevent an endless loop: getTok(p) @@ -3433,7 +3267,9 @@ proc parseAll(p: var Parser): PNode = parMessage(p, errInvalidIndentation % "parseAll") - addSkipped(p, result) + splitLookahead(p, a, 0, clPostfix) + addSkipped(p, result) + doAssert p.skipped.len == 0, "leftover comments and end of parse, report bug" setEndInfo() diff --git a/src/phrenderer.nim b/src/phrenderer.nim index 4afa035..dac8a17 100644 --- a/src/phrenderer.nim +++ b/src/phrenderer.nim @@ -28,6 +28,7 @@ const longIndentWid = IndentWidth * 2 MaxLineLen = 88 LineCommentColumn = 30 + blankAfterComplex = {nkObjectTy, nkEnumTy, nkTypeSection, nkProcDef .. nkIteratorDef} type TRenderTok* = object @@ -43,6 +44,7 @@ type TSrcGen* = object indent*: int lineLen*: int + line: int # The line where a token was last output col: int pos*: int # current position for iteration over the buffer idx*: int # current token index for iteration over the buffer @@ -51,6 +53,7 @@ type pendingNL*: int # negative if not active; else contains the # indentation value + pendingWhitespace: int comStack*: seq[PNode] # comment stack inside: set[Section] # Keeps track of contexts we are in @@ -171,6 +174,15 @@ proc addTok(g: var TSrcGen; kind: TokType; s: string; sym: PSym = nil) = if kind != tkSpaces: inc g.col, s.len + g.line += count(s, "\n") + +proc outputLine(g: TSrcGen): int = + ## The line it would be added if we were to add a token + g.line + + (if g.pendingNL >= 0: 1 + ord(g.pendingNewline) + else: 0 + ) + proc addPendingNL(g: var TSrcGen) = if g.pendingNL >= 0: let newlines = repeat("\n", 1 + ord(g.pendingNewline)) @@ -200,10 +212,10 @@ proc optNL(g: var TSrcGen; a, b: PNode) = g.lineLen = g.indent g.col = g.indent - g.pendingNewline = g.pendingNewline or a.info.line + 1 < b.info.line + g.pendingNewline = g.pendingNewline or a.endInfo.line + 1 < b.info.line -proc blankLine(g: var TSrcGen) = - g.pendingNewline = true +proc blankLine(g: var TSrcGen; v = true) = + g.pendingNewline = v or g.pendingNewline proc indentNL(g: var TSrcGen; wid = IndentWidth) = inc(g.indent, wid) @@ -244,7 +256,6 @@ proc put(g: var TSrcGen; kind: TokType; s: string; sym: PSym = nil) = proc putComment(g: var TSrcGen; s: string) = if s.len == 0: return - # var i = 0 # let hi = s.len - 1 # let isCode = (s.len >= 2) and (s[1] != ' ') @@ -381,7 +392,6 @@ proc litAux(g: TSrcGen; n: PNode; x: BiggestInt; size: int): string = result = typ.sym.name.s & '.' let enumfields = typ.n - # we need a slow linear search because of enums with holes: for e in items(enumfields): if e.sym.position == x: @@ -495,7 +505,6 @@ proc atom(g: TSrcGen; n: PNode): string = proc lcomma(g: TSrcGen; n: PNode; start: int = 0; theEnd: int = -1): int = assert(theEnd < 0) - # if n.prefix.len > 0 or n.mid.len > 0 or n.postfix.len > 0: # return MaxLineLen + 1 result = 0 @@ -512,7 +521,6 @@ proc lsons(g: TSrcGen; n: PNode; start: int = 0; theEnd: int = -1): int = assert(theEnd < 0) result = 0 - # if n.prefix.len > 0 or n.mid.len > 0 or n.postfix.len > 0: # return MaxLineLen + 1 for i in start .. n.len + theEnd: @@ -541,7 +549,6 @@ proc lsub(g: TSrcGen; n: PNode): int = # computes the length of a tree if isNil(n): return 0 - # if n.prefix.len > 0 or n.mid.len > 0 or n.postfix.len > 0: # return MaxLineLen + 1 if shouldRenderComment(g, n): @@ -885,7 +892,6 @@ proc gcommaAux( let oldInd = g.indent g.indent = ind - # If a full, comma-separate list fits on one line, go for it. If not, we put # each element on its own line unless it's a list of trivial things (so as to # avoid wasting significant vertical space on lists of numbers and the like) @@ -1036,7 +1042,6 @@ proc glist( proc gsection(g: var TSrcGen; n: PNode; kind: TokType; k: string) = if n.len == 0: return - # empty var sections are possible putWithSpace(g, kind, k) if n.len > 1 or n.mid.len > 0 or n[0].prefix.len > 0: @@ -1068,7 +1073,6 @@ proc gstmts(g: var TSrcGen; n: PNode; flags: SubFlags = {}; doIndent = true) = flags + {sfNoIndent} else: flags - # The original version tried to normalize newlines but this turned out to be # difficult to do whole-sale - we will thus retain some of the original # new-lines, normalizing only where a clear rule can be established @@ -1082,7 +1086,6 @@ proc gstmts(g: var TSrcGen; n: PNode; flags: SubFlags = {}; doIndent = true) = # probably add a newline even before control flow if i > 0: optNL(g, n[i - 1], n[i]) - # else: # optNL(g) if n[i].kind in {nkStmtList, nkStmtListExpr, nkStmtListType}: @@ -1204,7 +1207,6 @@ proc gproc(g: var TSrcGen; n: PNode) = if n[patternPos].kind != nkEmpty: gpattern(g, n[patternPos]) - # If there is no body, we don't need to indent the parameters as much let flags = if n[bodyPos].kind != nkEmpty: @@ -1225,7 +1227,6 @@ proc gproc(g: var TSrcGen; n: PNode) = gcoms(g) gstmts(g, n[bodyPos], flags) - blankLine(g) else: withIndent(g): gmids(g, n) @@ -1256,7 +1257,6 @@ proc gblock(g: var TSrcGen; n: PNode) = gsub(g, n[0]) else: put(g, tkBlock, "block") - # block stmt should have two children if n.len == 1: return @@ -1440,7 +1440,8 @@ proc gsub(g: var TSrcGen; n: PNode; flags: SubFlags; fromStmtList = false) = gstmts(g, n) return - + # When adding blanks after certain nodes, we only do so if there's a body + let currLine = g.outputLine() gprefixes(g, n) if shouldRenderComment(g, n): pushCom(g, n) @@ -1561,7 +1562,6 @@ proc gsub(g: var TSrcGen; n: PNode; flags: SubFlags; fromStmtList = false) = if i > 1: gcomma(g, n, 1, i - 1 - n.len, indentNL = IndentWidth, flags = {lfFirstSticky}) - # when parsing nkCommand, the compiler inserts `nkCall` to arguments if # ":" is present so it looks like we can skip the `do` here :/ this needs # deeper investigation - see also `nkPar` which sometimes removes the @@ -1603,9 +1603,16 @@ proc gsub(g: var TSrcGen; n: PNode; flags: SubFlags; fromStmtList = false) = of nkDotExpr: if isCustomLit(n): put(g, tkCustomLit, n[0].strVal) + gmids(g, n) gsub(g, n[1]) else: gsub(g, n[0]) + # Mids here are put on a new line, then the dot follows on yet another new + # line as dot parsing continues after a comment on a new line too! see + # clMid pickup point in phparser.dotExpr + if n.mid.len > 0: + g.optNL() + gmids(g, n) put(g, tkDot, ".") assert n.len == 2, $n.len @@ -1647,7 +1654,6 @@ proc gsub(g: var TSrcGen; n: PNode; flags: SubFlags; fromStmtList = false) = if n.len >= 1 and n[^1].kind != nkEmpty: put(g, tkSpaces, Space) putWithSpace(g, tkEquals, "=") - # TODO this is a hack to make `valueOr` in `let ... ` drop the `do` but # it is dubious if it is correct - there's a parsing ambiguity # in nkPar / nkCommand / nkCall that needs resolving @@ -1885,15 +1891,13 @@ proc gsub(g: var TSrcGen; n: PNode; flags: SubFlags; fromStmtList = false) = if n.len > 0: putWithSpace(g, tkObject, "object") g.inside(ObjectDef): - gsub(g, n[0]) - gsub(g, n[1]) + gsub(g, n[0]) # nkEmpty (unused) + gsub(g, n[1]) # nkOfInherit / nkEmpty withIndent(g): gmids(g, n) gcoms(g) - gsub(g, n[2]) - - blankLine(g) + gsub(g, n[2]) # fields else: put(g, tkObject, "object") of nkRecList: @@ -1902,7 +1906,15 @@ proc gsub(g: var TSrcGen; n: PNode; flags: SubFlags; fromStmtList = false) = gmids(g, n) for i in 0 ..< n.len: - optNL(g) + # The prefix check is here because if the previous line has a postfix + # and the next has a prefix and they are both doc comments, + # this might generate an invalid doc comment + # TODO this should only apply to doc comments and maybe needs deep + # comment inspection + if i > 0 and n[i].prefix.len == 0: + optNL(g, n[i - 1], n[i]) + else: + optNL(g) gsub(g, n[i]) gcoms(g) @@ -1944,7 +1956,6 @@ proc gsub(g: var TSrcGen; n: PNode; flags: SubFlags; fromStmtList = false) = gsonsNL(g, n, 1) gcoms(g) # BUGFIX: comment for the last enum field dedent(g) - blankLine(g) else: put(g, tkEnum, "enum") of nkEnumFieldDef: @@ -2004,7 +2015,6 @@ proc gsub(g: var TSrcGen; n: PNode; flags: SubFlags; fromStmtList = false) = gproc(g, n) of nkTypeSection: gsection(g, n, tkType, "type") - blankLine(g) of nkConstSection: gsection(g, n, tkConst, "const") of nkVarSection: @@ -2202,7 +2212,6 @@ proc gsub(g: var TSrcGen; n: PNode; flags: SubFlags; fromStmtList = false) = len(": ") + lsub(g, n[0]) else: 0 - # Semi-colon here is ugly but necessary for semantic equality, else we # get different groupings of nkIdentDefs and their descendants # TODO relaxing this to semantic equivalence would allow the use of `,` @@ -2242,6 +2251,9 @@ proc gsub(g: var TSrcGen; n: PNode; flags: SubFlags; fromStmtList = false) = if sfSkipPostfix notin flags: gpostfixes(g, n) + if n.kind in blankAfterComplex and currLine < g.line: + g.blankLine() + proc renderTree*(n: PNode; conf: ConfigRef = nil): string = if n == nil: return "" @@ -2256,7 +2268,6 @@ proc renderTree*(n: PNode; conf: ConfigRef = nil): string = conf , ) - # do not indent the initial statement list so that # writeFile("file.nim", repr n) # produces working Nim code: diff --git a/tests/after/empty.nim b/tests/after/a00empty.nim similarity index 100% rename from tests/after/empty.nim rename to tests/after/a00empty.nim diff --git a/tests/after/empty.nim.nph.yaml b/tests/after/a00empty.nim.nph.yaml similarity index 100% rename from tests/after/empty.nim.nph.yaml rename to tests/after/a00empty.nim.nph.yaml diff --git a/tests/after/comments.nim b/tests/after/comments.nim index 33697e3..3a40666 100644 --- a/tests/after/comments.nim +++ b/tests/after/comments.nim @@ -29,8 +29,6 @@ and this ]# ]# -x 324 # A comment after - template x() = ## A template doc comment try: @@ -39,6 +37,10 @@ template x() = mixin `$` # A comment after mixin echo 4 +type + # comment after type keyword + LonelyObject* = object + type SingleValueSetting* {.pure.} = enum ## \ @@ -51,9 +53,8 @@ type mm ## memory management selected FileSeekPos* = enum - fspEnd - ## Seek relative to end - # text file handling: + fspEnd ## Seek relative to end + # text file handling: ## Position relative to which seek should happen. # The values are ordered so that they match with stdio @@ -68,6 +69,11 @@ type # loooooooooooooooooooong comment past the max line length # and here + NewlineObject = object + field: int ## doc comment after field + ## doc comment continues, though not as a postfix - empty line after + field2: int ## just a doc again + Inherited = object of RootObj # inherited eol comment # inherited next line indent comments @@ -145,9 +151,9 @@ proc x() = proc x() = discard + ## indented doc comment for proc + ## that is long -## indented doc comment for proc -## that is long # before module import module # with a comment import module ## with a comment @@ -176,8 +182,8 @@ except: finally: # finally first dedent line discard - # finally last dedent line + for i in 0 .. 1: # for colon line # for first line discard @@ -191,6 +197,7 @@ else: # case else colon line discard f do -> int: # do colon line + # do first line discard discard @@ -204,7 +211,6 @@ let x = # lambda first line discard discard - while false: # while colon line # while first line discard @@ -217,8 +223,10 @@ discard Object( # object eol # object first line field: 0, # field line - field2: # Field colon next line - 42 # field colon line + field2: + # field colon line + # Field colon next line + 42 ) a = b @@ -230,6 +238,7 @@ proc ffff() = ## Doc comment after indented statement ## needs to be double + abc and # dedented comment in infix def @@ -248,8 +257,8 @@ a(b = c # comment after keyword parameter ) a(b = c) - # dedented comment after keyword parameter + {.pragma # comment here .} @@ -269,11 +278,6 @@ discard # discard first line 54 # discard value -block: - # also after call - # comment between the dots - f.x.z().d() - proc f(): bool = ## Comment here ## another @@ -299,3 +303,48 @@ command "a", "b", "c" # command eol comment command "first arg", # first arg comment "second arg", # second arg comment "third arg" # third arg comment + +command "first arg" +# comment after command + +command 234 # command after ind + +when false: + command "first arg" + # comment after command nest + +when false: + command a.b + # comment after command dotexpr nest + +call() # call eol + +echo dotexpr.dot # after dotexpr in command + +# between two dotepxrs + +dotexpr +# between dotexpr and dotonnewline +.dotonnewline + +if true: + echo dotexpr.dot # after dotexpr in command ind + # between two dotepxrs ind + + dotexpr + # between dotexpr and dotonnewline ind + .dotonnewline + +block: + f.x + # comment between the dots + .z() + # also after call + .d() # far eol of dotexpr + + # after dotexpr ind + +block: + # no whitespace between the next two multilines + functionCall(param) + functionCall(param) diff --git a/tests/after/comments.nim.nph.yaml b/tests/after/comments.nim.nph.yaml index 4de49bb..3916338 100644 --- a/tests/after/comments.nim.nph.yaml +++ b/tests/after/comments.nim.nph.yaml @@ -28,14 +28,6 @@ sons: "comment": "##[\u000A these also come in doc variants\u000A]##" - kind: "nkCommentStmt" "comment": "#[\u000A #[\u000A they can be nested\u000A ]#\u000A]#" - - kind: "nkCommand" - sons: - - kind: "nkIdent" - ident: "x" - - kind: "nkIntLit" - intVal: 324 - postfix: - - "# A comment after" - kind: "nkTemplateDef" sons: - kind: "nkIdent" @@ -48,9 +40,9 @@ sons: - kind: "nkEmpty" - kind: "nkEmpty" - kind: "nkStmtList" - prefix: - - "## A template doc comment" sons: + - kind: "nkCommentStmt" + "comment": "## A template doc comment" - kind: "nkTryStmt" sons: - kind: "nkStmtList" @@ -78,6 +70,24 @@ sons: ident: "echo" - kind: "nkIntLit" intVal: 4 + - kind: "nkTypeSection" + sons: + - kind: "nkTypeDef" + prefix: + - "# comment after type keyword" + sons: + - kind: "nkPostfix" + sons: + - kind: "nkIdent" + ident: "*" + - kind: "nkIdent" + ident: "LonelyObject" + - kind: "nkEmpty" + - kind: "nkObjectTy" + sons: + - kind: "nkEmpty" + - kind: "nkEmpty" + - kind: "nkEmpty" - kind: "nkTypeSection" sons: - kind: "nkTypeDef" @@ -140,9 +150,9 @@ sons: ident: "fspEnd" postfix: - "## Seek relative to end" - - "# text file handling:" - kind: "nkTypeDef" prefix: + - "# text file handling:" - "## Position relative to which seek should happen." - "# The values are ordered so that they match with stdio" - "# SEEK_SET, SEEK_CUR and SEEK_END respectively." @@ -191,6 +201,37 @@ sons: - kind: "nkTypeDef" prefix: - "# and here" + sons: + - kind: "nkIdent" + ident: "NewlineObject" + - kind: "nkEmpty" + - kind: "nkObjectTy" + sons: + - kind: "nkEmpty" + - kind: "nkEmpty" + - kind: "nkRecList" + sons: + - kind: "nkIdentDefs" + sons: + - kind: "nkIdent" + ident: "field" + - kind: "nkIdent" + ident: "int" + - kind: "nkEmpty" + postfix: + - "## doc comment after field" + - kind: "nkIdentDefs" + prefix: + - "## doc comment continues, though not as a postfix - empty line after" + sons: + - kind: "nkIdent" + ident: "field2" + - kind: "nkIdent" + ident: "int" + - kind: "nkEmpty" + postfix: + - "## just a doc again" + - kind: "nkTypeDef" sons: - kind: "nkIdent" ident: "Inherited" @@ -228,6 +269,8 @@ sons: - kind: "nkRecList" sons: - kind: "nkRecCase" + mid: + - "# casetype eol" sons: - kind: "nkIdentDefs" sons: @@ -236,8 +279,6 @@ sons: - kind: "nkIdent" ident: "bool" - kind: "nkEmpty" - postfix: - - "# casetype eol" - kind: "nkOfBranch" mid: - "# of eol" @@ -399,9 +440,9 @@ sons: - kind: "nkIdent" ident: "somecond" - kind: "nkStmtList" - prefix: - - "# when first line" sons: + - kind: "nkCommentStmt" + "comment": "# when first line" - kind: "nkDiscardStmt" sons: - kind: "nkEmpty" @@ -412,9 +453,9 @@ sons: - "# else colon line" sons: - kind: "nkStmtList" - prefix: - - "# else first line" sons: + - kind: "nkCommentStmt" + "comment": "# else first line" - kind: "nkDiscardStmt" sons: - kind: "nkEmpty" @@ -425,18 +466,18 @@ sons: - kind: "nkIdent" ident: "true" - kind: "nkStmtList" - prefix: - - "# if next line" sons: + - kind: "nkCommentStmt" + "comment": "# if next line" - kind: "nkDiscardStmt" sons: - kind: "nkEmpty" - kind: "nkElse" sons: - kind: "nkStmtList" - prefix: - - "# else next line" sons: + - kind: "nkCommentStmt" + "comment": "# else next line" - kind: "nkDiscardStmt" sons: - kind: "nkEmpty" @@ -469,9 +510,9 @@ sons: - kind: "nkIdent" ident: "true" - kind: "nkStmtList" - prefix: - - "# if dedented colon line" sons: + - kind: "nkCommentStmt" + "comment": "# if dedented colon line" - kind: "nkDiscardStmt" sons: - kind: "nkEmpty" @@ -518,9 +559,9 @@ sons: - kind: "nkEmpty" - kind: "nkEmpty" - kind: "nkStmtList" - prefix: - - "# proc first line" sons: + - kind: "nkCommentStmt" + "comment": "# proc first line" - kind: "nkDiscardStmt" sons: - kind: "nkEmpty" @@ -536,9 +577,9 @@ sons: - kind: "nkEmpty" - kind: "nkEmpty" - kind: "nkStmtList" - prefix: - - "## A proc doc comment" sons: + - kind: "nkCommentStmt" + "comment": "## A proc doc comment" - kind: "nkIfStmt" sons: - kind: "nkElifBranch" @@ -590,11 +631,13 @@ sons: - kind: "nkDiscardStmt" sons: - kind: "nkEmpty" + - kind: "nkCommentStmt" + "comment": "## indented doc comment for proc" + - kind: "nkCommentStmt" + "comment": "## that is long" + - kind: "nkCommentStmt" + "comment": "# before module" - kind: "nkImportStmt" - prefix: - - "## indented doc comment for proc" - - "## that is long" - - "# before module" sons: - kind: "nkIdent" ident: "module" @@ -611,9 +654,9 @@ sons: - "# try colon line" sons: - kind: "nkStmtList" - prefix: - - "# try first line" sons: + - kind: "nkCommentStmt" + "comment": "# try first line" - kind: "nkDiscardStmt" sons: - kind: "nkEmpty" @@ -624,9 +667,9 @@ sons: - "# except colon line" sons: - kind: "nkStmtList" - prefix: - - "# except first line" sons: + - kind: "nkCommentStmt" + "comment": "# except first line" - kind: "nkDiscardStmt" sons: - kind: "nkEmpty" @@ -637,9 +680,9 @@ sons: - "# Finally colon line" sons: - kind: "nkStmtList" - prefix: - - "# finally first line" sons: + - kind: "nkCommentStmt" + "comment": "# finally first line" - kind: "nkDiscardStmt" sons: - kind: "nkEmpty" @@ -648,9 +691,9 @@ sons: - kind: "nkTryStmt" sons: - kind: "nkStmtList" - prefix: - - "# try first dedent line" sons: + - kind: "nkCommentStmt" + "comment": "# try first dedent line" - kind: "nkCall" sons: - kind: "nkIdent" @@ -660,9 +703,9 @@ sons: - "# try last dedent line" sons: - kind: "nkStmtList" - prefix: - - "# except dedent first line" sons: + - kind: "nkCommentStmt" + "comment": "# except dedent first line" - kind: "nkDiscardStmt" sons: - kind: "nkEmpty" @@ -671,15 +714,15 @@ sons: - "# except dedent last line" sons: - kind: "nkStmtList" - prefix: - - "# finally first dedent line" sons: + - kind: "nkCommentStmt" + "comment": "# finally first dedent line" - kind: "nkDiscardStmt" sons: - kind: "nkEmpty" + - kind: "nkCommentStmt" + "comment": "# finally last dedent line" - kind: "nkForStmt" - prefix: - - "# finally last dedent line" mid: - "# for colon line" sons: @@ -694,9 +737,9 @@ sons: - kind: "nkIntLit" intVal: 1 - kind: "nkStmtList" - prefix: - - "# for first line" sons: + - kind: "nkCommentStmt" + "comment": "# for first line" - kind: "nkDiscardStmt" sons: - kind: "nkEmpty" @@ -713,9 +756,9 @@ sons: - kind: "nkIdent" ident: "true" - kind: "nkStmtList" - prefix: - - "# of first line" sons: + - kind: "nkCommentStmt" + "comment": "# of first line" - kind: "nkDiscardStmt" sons: - kind: "nkEmpty" @@ -724,9 +767,9 @@ sons: - "# case else colon line" sons: - kind: "nkStmtList" - prefix: - - "# case else first line" sons: + - kind: "nkCommentStmt" + "comment": "# case else first line" - kind: "nkDiscardStmt" sons: - kind: "nkEmpty" @@ -749,6 +792,8 @@ sons: - kind: "nkEmpty" - kind: "nkStmtList" sons: + - kind: "nkCommentStmt" + "comment": "# do first line" - kind: "nkDiscardStmt" sons: - kind: "nkEmpty" @@ -761,9 +806,9 @@ sons: sons: - kind: "nkEmpty" - kind: "nkStmtList" - prefix: - - "# block first line" sons: + - kind: "nkCommentStmt" + "comment": "# block first line" - kind: "nkDiscardStmt" sons: - kind: "nkEmpty" @@ -791,9 +836,9 @@ sons: - kind: "nkEmpty" - kind: "nkEmpty" - kind: "nkStmtList" - prefix: - - "# lambda first line" sons: + - kind: "nkCommentStmt" + "comment": "# lambda first line" - kind: "nkDiscardStmt" sons: - kind: "nkEmpty" @@ -807,9 +852,9 @@ sons: - kind: "nkIdent" ident: "false" - kind: "nkStmtList" - prefix: - - "# while first line" sons: + - kind: "nkCommentStmt" + "comment": "# while first line" - kind: "nkDiscardStmt" sons: - kind: "nkEmpty" @@ -818,9 +863,9 @@ sons: - "# static colon line" sons: - kind: "nkStmtList" - prefix: - - "# static first line" sons: + - kind: "nkCommentStmt" + "comment": "# static first line" - kind: "nkDiscardStmt" sons: - kind: "nkEmpty" @@ -846,10 +891,10 @@ sons: - kind: "nkIdent" ident: "field2" - kind: "nkIntLit" + prefix: + - "# field colon line" + - "# Field colon next line" intVal: 42 - postfix: - - "# Field colon next line" - - "# field colon line" - kind: "nkAsgn" sons: - kind: "nkIdent" @@ -881,6 +926,10 @@ sons: ident: "result" - kind: "nkIdent" ident: "add" + - kind: "nkCommentStmt" + "comment": "## Doc comment after indented statement" + - kind: "nkCommentStmt" + "comment": "## needs to be double" - kind: "nkInfix" sons: - kind: "nkIdent" @@ -888,9 +937,6 @@ sons: postfix: - "# dedented comment in infix" - kind: "nkIdent" - prefix: - - "## Doc comment after indented statement" - - "## needs to be double" ident: "abc" - kind: "nkIdent" ident: "def" @@ -964,9 +1010,9 @@ sons: ident: "b" - kind: "nkIdent" ident: "c" + - kind: "nkCommentStmt" + "comment": "# dedented comment after keyword parameter" - kind: "nkPragma" - prefix: - - "# dedented comment after keyword parameter" sons: - kind: "nkIdent" ident: "pragma" @@ -1035,32 +1081,6 @@ sons: intVal: 54 postfix: - "# discard value" - - kind: "nkBlockStmt" - sons: - - kind: "nkEmpty" - - kind: "nkStmtList" - prefix: - - "# also after call" - - "# comment between the dots" - sons: - - kind: "nkCall" - sons: - - kind: "nkDotExpr" - sons: - - kind: "nkCall" - sons: - - kind: "nkDotExpr" - sons: - - kind: "nkDotExpr" - sons: - - kind: "nkIdent" - ident: "f" - - kind: "nkIdent" - ident: "x" - - kind: "nkIdent" - ident: "z" - - kind: "nkIdent" - ident: "d" - kind: "nkProcDef" sons: - kind: "nkIdent" @@ -1074,10 +1094,11 @@ sons: - kind: "nkEmpty" - kind: "nkEmpty" - kind: "nkStmtList" - prefix: - - "## Comment here" - - "## another" sons: + - kind: "nkCommentStmt" + "comment": "## Comment here" + - kind: "nkCommentStmt" + "comment": "## another" - kind: "nkPar" sons: - kind: "nkInfix" @@ -1101,10 +1122,11 @@ sons: - kind: "nkEmpty" - kind: "nkEmpty" - kind: "nkStmtList" - prefix: - - "## Comment here" - - "## another" sons: + - kind: "nkCommentStmt" + "comment": "## Comment here" + - kind: "nkCommentStmt" + "comment": "## another" - kind: "nkIfStmt" sons: - kind: "nkElifBranch" @@ -1118,10 +1140,11 @@ sons: - kind: "nkElse" sons: - kind: "nkStmtList" - prefix: - - "## comment" - - "## comment 2" sons: + - kind: "nkCommentStmt" + "comment": "## comment" + - kind: "nkCommentStmt" + "comment": "## comment 2" - kind: "nkIfStmt" sons: - kind: "nkElifBranch" @@ -1129,10 +1152,11 @@ sons: - kind: "nkIdent" ident: "true" - kind: "nkStmtList" - prefix: - - "## comment" - - "## comment 2" sons: + - kind: "nkCommentStmt" + "comment": "## comment" + - kind: "nkCommentStmt" + "comment": "## comment 2" - kind: "nkPar" sons: - kind: "nkInfix" @@ -1176,4 +1200,164 @@ sons: - kind: "nkStrLit" strVal: "third arg" postfix: - - "# third arg comment" \ No newline at end of file + - "# third arg comment" + - kind: "nkCommand" + sons: + - kind: "nkIdent" + ident: "command" + - kind: "nkStrLit" + strVal: "first arg" + - kind: "nkCommentStmt" + "comment": "# comment after command" + - kind: "nkCommand" + sons: + - kind: "nkIdent" + ident: "command" + - kind: "nkIntLit" + intVal: 234 + postfix: + - "# command after ind" + - kind: "nkWhenStmt" + sons: + - kind: "nkElifBranch" + sons: + - kind: "nkIdent" + ident: "false" + - kind: "nkStmtList" + sons: + - kind: "nkCommand" + sons: + - kind: "nkIdent" + ident: "command" + - kind: "nkStrLit" + strVal: "first arg" + - kind: "nkCommentStmt" + "comment": "# comment after command nest" + - kind: "nkWhenStmt" + sons: + - kind: "nkElifBranch" + sons: + - kind: "nkIdent" + ident: "false" + - kind: "nkStmtList" + sons: + - kind: "nkCommand" + sons: + - kind: "nkIdent" + ident: "command" + - kind: "nkDotExpr" + sons: + - kind: "nkIdent" + ident: "a" + - kind: "nkIdent" + ident: "b" + - kind: "nkCommentStmt" + "comment": "# comment after command dotexpr nest" + - kind: "nkCall" + sons: + - kind: "nkIdent" + ident: "call" + postfix: + - "# call eol" + - kind: "nkCommand" + sons: + - kind: "nkIdent" + ident: "echo" + - kind: "nkDotExpr" + sons: + - kind: "nkIdent" + ident: "dotexpr" + - kind: "nkIdent" + ident: "dot" + postfix: + - "# after dotexpr in command" + - kind: "nkCommentStmt" + "comment": "# between two dotepxrs" + - kind: "nkDotExpr" + mid: + - "# between dotexpr and dotonnewline" + sons: + - kind: "nkIdent" + ident: "dotexpr" + - kind: "nkIdent" + ident: "dotonnewline" + - kind: "nkIfStmt" + sons: + - kind: "nkElifBranch" + sons: + - kind: "nkIdent" + ident: "true" + - kind: "nkStmtList" + sons: + - kind: "nkCommand" + sons: + - kind: "nkIdent" + ident: "echo" + - kind: "nkDotExpr" + sons: + - kind: "nkIdent" + ident: "dotexpr" + - kind: "nkIdent" + ident: "dot" + postfix: + - "# after dotexpr in command ind" + - kind: "nkCommentStmt" + "comment": "# between two dotepxrs ind" + - kind: "nkDotExpr" + mid: + - "# between dotexpr and dotonnewline ind" + sons: + - kind: "nkIdent" + ident: "dotexpr" + - kind: "nkIdent" + ident: "dotonnewline" + - kind: "nkBlockStmt" + sons: + - kind: "nkEmpty" + - kind: "nkStmtList" + sons: + - kind: "nkCall" + sons: + - kind: "nkDotExpr" + mid: + - "# also after call" + sons: + - kind: "nkCall" + sons: + - kind: "nkDotExpr" + mid: + - "# comment between the dots" + sons: + - kind: "nkDotExpr" + sons: + - kind: "nkIdent" + ident: "f" + - kind: "nkIdent" + ident: "x" + - kind: "nkIdent" + ident: "z" + - kind: "nkIdent" + ident: "d" + postfix: + - "# far eol of dotexpr" + - kind: "nkCommentStmt" + "comment": "# after dotexpr ind" + - kind: "nkBlockStmt" + sons: + - kind: "nkEmpty" + - kind: "nkStmtList" + sons: + - kind: "nkCommentStmt" + "comment": "# no whitespace between the next two multilines" + - kind: "nkCall" + sons: + - kind: "nkIdent" + ident: "functionCall" + - kind: "nkIdent" + ident: "param" + - kind: "nkCall" + sons: + - kind: "nkIdent" + ident: "functionCall" + - kind: "nkIdent" + ident: "param" \ No newline at end of file diff --git a/tests/after/exprs.nim b/tests/after/exprs.nim index 17304c1..e49b2aa 100644 --- a/tests/after/exprs.nim +++ b/tests/after/exprs.nim @@ -31,6 +31,9 @@ template ttt*(): untyped = discard xxxx "arg": yyy +# TODO +# (try: (discard rOk.tryError(); false) except ResultError[int]: true) + res.add fff do: yy() diff --git a/tests/after/exprs.nim.nph.yaml b/tests/after/exprs.nim.nph.yaml index c8f8bdd..cca62e6 100644 --- a/tests/after/exprs.nim.nph.yaml +++ b/tests/after/exprs.nim.nph.yaml @@ -112,9 +112,9 @@ sons: - kind: "nkDiscardStmt" sons: - kind: "nkEmpty" + - kind: "nkCommentStmt" + "comment": "# needs spaces" - kind: "nkForStmt" - prefix: - - "# needs spaces" sons: - kind: "nkIdent" ident: "a" @@ -168,9 +168,9 @@ sons: sons: - kind: "nkIdent" ident: "xxx" + - kind: "nkCommentStmt" + "comment": "# command syntax with colon" - kind: "nkDiscardStmt" - prefix: - - "# command syntax with colon" sons: - kind: "nkCommand" sons: @@ -182,6 +182,10 @@ sons: sons: - kind: "nkIdent" ident: "yyy" + - kind: "nkCommentStmt" + "comment": "# TODO" + - kind: "nkCommentStmt" + "comment": "# (try: (discard rOk.tryError(); false) except ResultError[int]: true)" - kind: "nkCommand" sons: - kind: "nkDotExpr" @@ -200,11 +204,13 @@ sons: sons: - kind: "nkIdent" ident: "yy" + - kind: "nkCommentStmt" + "comment": "# we don\'t what `do` in let but need it in the above commend - this needs" + - kind: "nkCommentStmt" + "comment": "# more investigation - this is important for templates calls like Result.valueOr" + - kind: "nkCommentStmt" + "comment": "# which become ugly otherwise" - kind: "nkLetSection" - prefix: - - "# we don\'t what `do` in let but need it in the above commend - this needs" - - "# more investigation - this is important for templates calls like Result.valueOr" - - "# which become ugly otherwise" sons: - kind: "nkIdentDefs" sons: diff --git a/tests/after/procs.nim b/tests/after/procs.nim index deda830..b77ac4d 100644 --- a/tests/after/procs.nim +++ b/tests/after/procs.nim @@ -44,10 +44,12 @@ proc aaaaa[A, B, C]( aaaaaaaaaaaaaaaaaaaaaaaaaaa, bbbbbbbbbbbbbbbbbbb, cccccccccccccccc, ddddddddddddddddddd: int; ) + proc aaaa*[ Aaaaaaaaaaaaaaaaaaaaaaaa, Bbbbbbbbbbbbbbbbbbbbbbbbbb, Cccccccccccccccccccccc, Dddddddddddddddddddddd ](v: int) + proc aaaaaaaaa*[ Aaaaaaaaaaaaaaaaaaaaaaaa, Bbbbbbbbbbbbbbbbbbbbbbbbbb, Cccccccccccccccccccccc, Dddddddddddddddddddddd @@ -60,6 +62,7 @@ proc aaaaaaaa[T: Aaaa](v: int) proc aaaaaaaa[ Tttttttttttttttttttttttttttttttttttttttttt: Aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa ](v: int) + proc aaaaaaaa[T: Aaaa; S: static int](v: int) proc aaaaaaaa[ T: @@ -93,17 +96,11 @@ aasdcsaa( ) type Ap = proc() - type Bp = proc - type Cp = proc(v: int) - type Dp = proc() {.nimcall.} - type Ep = proc {.nimcall.} - type Aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa = proc {.bbbbbbbbbbbbbbbbbbbbbbbb.} - type Aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa = proc(aaaaaaaa: Bbbbbbb; ccccccccc: Dddddddddddd; eeeeee: Ffffff) diff --git a/tests/after/types.nim b/tests/after/types.nim index 4b340fa..8ecd396 100644 --- a/tests/after/types.nim +++ b/tests/after/types.nim @@ -1,7 +1,6 @@ type A = int type B = A | B - type Bbbbbbbbbbbbbbbbbbbbbbbbbbbbb = Aaaaaaaaaaaaaa | Bbbbbbbbbbbbbbbbb | Cccccccccccccccccccccc | Dddddddddddddddddddddd diff --git a/tests/before/empty.nim b/tests/before/a00empty.nim similarity index 100% rename from tests/before/empty.nim rename to tests/before/a00empty.nim diff --git a/tests/before/empty.nim.nph.yaml b/tests/before/a00empty.nim.nph.yaml similarity index 100% rename from tests/before/empty.nim.nph.yaml rename to tests/before/a00empty.nim.nph.yaml diff --git a/tests/before/comments.nim b/tests/before/comments.nim index c0c8dbe..4d3598f 100644 --- a/tests/before/comments.nim +++ b/tests/before/comments.nim @@ -30,9 +30,6 @@ and this ]# ]# -x 324 - # A comment after - template x = ## A template doc comment try: @@ -41,6 +38,10 @@ template x = mixin `$` # A comment after mixin echo 4 +type + # comment after type keyword + LonelyObject* = object + type SingleValueSetting* {.pure.} = enum ## \ ## settings resulting in a single string value @@ -65,6 +66,12 @@ type fiiiiiiiiiiiiiiiiiiiiiiiieeeeeeeeeld: int # loooooooooooooooooooong comment past the max line length # and here + NewlineObject = object + field: int ## doc comment after field + ## doc comment continues, though not as a postfix - empty line after + + field2: int ## just a doc again + Inherited = object of RootObj # inherited eol comment # inherited next line indent comments f: int @@ -270,14 +277,6 @@ discard # discard eol # discard first line 54 # discard value -block: - f - .x - # comment between the dots - .z() - # also after call - .d() - proc f: bool = ## Comment here ## another @@ -298,9 +297,60 @@ proc f: bool = else: false - command "a", "b", "c" # command eol comment command "first arg", # first arg comment "second arg", # second arg comment "third arg" # third arg comment + +command "first arg" +# comment after command + +command 234 + # command after ind + +when false: + command "first arg" + # comment after command nest + +when false: + command a.b + # comment after command dotexpr nest + +call() # call eol + +echo dotexpr.dot # after dotexpr in command + +# between two dotepxrs + +dotexpr +# between dotexpr and dotonnewline +.dotonnewline + +if true: + echo dotexpr.dot # after dotexpr in command ind + # between two dotepxrs ind + dotexpr + # between dotexpr and dotonnewline ind + .dotonnewline + +block: + f + .x + # comment between the dots + .z() + # also after call + .d() # far eol of dotexpr + + # after dotexpr ind + + + +block: + # no whitespace between the next two multilines + functionCall( + param + ) + functionCall( + param + ) \ No newline at end of file diff --git a/tests/before/comments.nim.nph.yaml b/tests/before/comments.nim.nph.yaml index 8e0629f..7b94d7e 100644 --- a/tests/before/comments.nim.nph.yaml +++ b/tests/before/comments.nim.nph.yaml @@ -28,14 +28,6 @@ sons: "comment": "##[\u000A these also come in doc variants\u000A]##" - kind: "nkCommentStmt" "comment": "#[\u000A #[\u000A they can be nested\u000A ]#\u000A]#" - - kind: "nkCommand" - sons: - - kind: "nkIdent" - ident: "x" - - kind: "nkIntLit" - intVal: 324 - postfix: - - "# A comment after" - kind: "nkTemplateDef" sons: - kind: "nkIdent" @@ -48,9 +40,9 @@ sons: - kind: "nkEmpty" - kind: "nkEmpty" - kind: "nkStmtList" - prefix: - - "## A template doc comment" sons: + - kind: "nkCommentStmt" + "comment": "## A template doc comment" - kind: "nkTryStmt" sons: - kind: "nkStmtList" @@ -78,6 +70,24 @@ sons: ident: "echo" - kind: "nkIntLit" intVal: 4 + - kind: "nkTypeSection" + sons: + - kind: "nkTypeDef" + prefix: + - "# comment after type keyword" + sons: + - kind: "nkPostfix" + sons: + - kind: "nkIdent" + ident: "*" + - kind: "nkIdent" + ident: "LonelyObject" + - kind: "nkEmpty" + - kind: "nkObjectTy" + sons: + - kind: "nkEmpty" + - kind: "nkEmpty" + - kind: "nkEmpty" - kind: "nkTypeSection" sons: - kind: "nkTypeDef" @@ -140,7 +150,8 @@ sons: ident: "fspEnd" postfix: - "## Seek relative to end" - - "# text file handling:" + postfix: + - "# text file handling:" - kind: "nkTypeDef" prefix: - "## Position relative to which seek should happen." @@ -191,6 +202,37 @@ sons: - kind: "nkTypeDef" prefix: - "# and here" + sons: + - kind: "nkIdent" + ident: "NewlineObject" + - kind: "nkEmpty" + - kind: "nkObjectTy" + sons: + - kind: "nkEmpty" + - kind: "nkEmpty" + - kind: "nkRecList" + sons: + - kind: "nkIdentDefs" + sons: + - kind: "nkIdent" + ident: "field" + - kind: "nkIdent" + ident: "int" + - kind: "nkEmpty" + postfix: + - "## doc comment after field" + - kind: "nkIdentDefs" + prefix: + - "## doc comment continues, though not as a postfix - empty line after" + sons: + - kind: "nkIdent" + ident: "field2" + - kind: "nkIdent" + ident: "int" + - kind: "nkEmpty" + postfix: + - "## just a doc again" + - kind: "nkTypeDef" sons: - kind: "nkIdent" ident: "Inherited" @@ -228,6 +270,8 @@ sons: - kind: "nkRecList" sons: - kind: "nkRecCase" + mid: + - "# casetype eol" sons: - kind: "nkIdentDefs" sons: @@ -236,8 +280,6 @@ sons: - kind: "nkIdent" ident: "bool" - kind: "nkEmpty" - postfix: - - "# casetype eol" - kind: "nkOfBranch" mid: - "# of eol" @@ -401,9 +443,9 @@ sons: - kind: "nkIdent" ident: "somecond" - kind: "nkStmtList" - prefix: - - "# when first line" sons: + - kind: "nkCommentStmt" + "comment": "# when first line" - kind: "nkDiscardStmt" sons: - kind: "nkEmpty" @@ -414,9 +456,9 @@ sons: - "# else colon line" sons: - kind: "nkStmtList" - prefix: - - "# else first line" sons: + - kind: "nkCommentStmt" + "comment": "# else first line" - kind: "nkDiscardStmt" sons: - kind: "nkEmpty" @@ -427,18 +469,18 @@ sons: - kind: "nkIdent" ident: "true" - kind: "nkStmtList" - prefix: - - "# if next line" sons: + - kind: "nkCommentStmt" + "comment": "# if next line" - kind: "nkDiscardStmt" sons: - kind: "nkEmpty" - kind: "nkElse" sons: - kind: "nkStmtList" - prefix: - - "# else next line" sons: + - kind: "nkCommentStmt" + "comment": "# else next line" - kind: "nkDiscardStmt" sons: - kind: "nkEmpty" @@ -471,9 +513,9 @@ sons: - kind: "nkIdent" ident: "true" - kind: "nkStmtList" - prefix: - - "# if dedented colon line" sons: + - kind: "nkCommentStmt" + "comment": "# if dedented colon line" - kind: "nkDiscardStmt" sons: - kind: "nkEmpty" @@ -520,9 +562,9 @@ sons: - kind: "nkEmpty" - kind: "nkEmpty" - kind: "nkStmtList" - prefix: - - "# proc first line" sons: + - kind: "nkCommentStmt" + "comment": "# proc first line" - kind: "nkDiscardStmt" sons: - kind: "nkEmpty" @@ -538,9 +580,9 @@ sons: - kind: "nkEmpty" - kind: "nkEmpty" - kind: "nkStmtList" - prefix: - - "## A proc doc comment" sons: + - kind: "nkCommentStmt" + "comment": "## A proc doc comment" - kind: "nkIfStmt" sons: - kind: "nkElifBranch" @@ -592,11 +634,12 @@ sons: - kind: "nkDiscardStmt" sons: - kind: "nkEmpty" - - kind: "nkImportStmt" - prefix: + postfix: - "## indented doc comment for proc" - "## that is long" - - "# before module" + - kind: "nkCommentStmt" + "comment": "# before module" + - kind: "nkImportStmt" sons: - kind: "nkIdent" ident: "module" @@ -613,9 +656,9 @@ sons: - "# try colon line" sons: - kind: "nkStmtList" - prefix: - - "# try first line" sons: + - kind: "nkCommentStmt" + "comment": "# try first line" - kind: "nkDiscardStmt" sons: - kind: "nkEmpty" @@ -626,9 +669,9 @@ sons: - "# except colon line" sons: - kind: "nkStmtList" - prefix: - - "# except first line" sons: + - kind: "nkCommentStmt" + "comment": "# except first line" - kind: "nkDiscardStmt" sons: - kind: "nkEmpty" @@ -639,9 +682,9 @@ sons: - "# Finally colon line" sons: - kind: "nkStmtList" - prefix: - - "# finally first line" sons: + - kind: "nkCommentStmt" + "comment": "# finally first line" - kind: "nkDiscardStmt" sons: - kind: "nkEmpty" @@ -650,9 +693,9 @@ sons: - kind: "nkTryStmt" sons: - kind: "nkStmtList" - prefix: - - "# try first dedent line" sons: + - kind: "nkCommentStmt" + "comment": "# try first dedent line" - kind: "nkCall" sons: - kind: "nkIdent" @@ -662,9 +705,9 @@ sons: - "# try last dedent line" sons: - kind: "nkStmtList" - prefix: - - "# except dedent first line" sons: + - kind: "nkCommentStmt" + "comment": "# except dedent first line" - kind: "nkDiscardStmt" sons: - kind: "nkEmpty" @@ -673,15 +716,15 @@ sons: - "# except dedent last line" sons: - kind: "nkStmtList" - prefix: - - "# finally first dedent line" sons: + - kind: "nkCommentStmt" + "comment": "# finally first dedent line" - kind: "nkDiscardStmt" sons: - kind: "nkEmpty" + - kind: "nkCommentStmt" + "comment": "# finally last dedent line" - kind: "nkForStmt" - prefix: - - "# finally last dedent line" mid: - "# for colon line" sons: @@ -696,9 +739,9 @@ sons: - kind: "nkIntLit" intVal: 1 - kind: "nkStmtList" - prefix: - - "# for first line" sons: + - kind: "nkCommentStmt" + "comment": "# for first line" - kind: "nkDiscardStmt" sons: - kind: "nkEmpty" @@ -715,9 +758,9 @@ sons: - kind: "nkIdent" ident: "true" - kind: "nkStmtList" - prefix: - - "# of first line" sons: + - kind: "nkCommentStmt" + "comment": "# of first line" - kind: "nkDiscardStmt" sons: - kind: "nkEmpty" @@ -726,9 +769,9 @@ sons: - "# case else colon line" sons: - kind: "nkStmtList" - prefix: - - "# case else first line" sons: + - kind: "nkCommentStmt" + "comment": "# case else first line" - kind: "nkDiscardStmt" sons: - kind: "nkEmpty" @@ -751,6 +794,8 @@ sons: - kind: "nkEmpty" - kind: "nkStmtList" sons: + - kind: "nkCommentStmt" + "comment": "# do first line" - kind: "nkDiscardStmt" sons: - kind: "nkEmpty" @@ -763,9 +808,9 @@ sons: sons: - kind: "nkEmpty" - kind: "nkStmtList" - prefix: - - "# block first line" sons: + - kind: "nkCommentStmt" + "comment": "# block first line" - kind: "nkDiscardStmt" sons: - kind: "nkEmpty" @@ -793,9 +838,9 @@ sons: - kind: "nkEmpty" - kind: "nkEmpty" - kind: "nkStmtList" - prefix: - - "# lambda first line" sons: + - kind: "nkCommentStmt" + "comment": "# lambda first line" - kind: "nkDiscardStmt" sons: - kind: "nkEmpty" @@ -809,9 +854,9 @@ sons: - kind: "nkIdent" ident: "false" - kind: "nkStmtList" - prefix: - - "# while first line" sons: + - kind: "nkCommentStmt" + "comment": "# while first line" - kind: "nkDiscardStmt" sons: - kind: "nkEmpty" @@ -820,9 +865,9 @@ sons: - "# static colon line" sons: - kind: "nkStmtList" - prefix: - - "# static first line" sons: + - kind: "nkCommentStmt" + "comment": "# static first line" - kind: "nkDiscardStmt" sons: - kind: "nkEmpty" @@ -850,10 +895,9 @@ sons: ident: "field2" - kind: "nkIntLit" prefix: + - "# field colon line" - "# Field colon next line" intVal: 42 - postfix: - - "# field colon line" - kind: "nkAsgn" sons: - kind: "nkIdent" @@ -885,14 +929,15 @@ sons: ident: "result" - kind: "nkIdent" ident: "add" + - kind: "nkCommentStmt" + "comment": "## Doc comment after indented statement" + - kind: "nkCommentStmt" + "comment": "## needs to be double" - kind: "nkInfix" sons: - kind: "nkIdent" ident: "and" - kind: "nkIdent" - prefix: - - "## Doc comment after indented statement" - - "## needs to be double" ident: "abc" - kind: "nkIdent" prefix: @@ -968,9 +1013,9 @@ sons: ident: "b" - kind: "nkIdent" ident: "c" + - kind: "nkCommentStmt" + "comment": "# dedented comment after keyword parameter" - kind: "nkPragma" - prefix: - - "# dedented comment after keyword parameter" sons: - kind: "nkIdent" ident: "pragma" @@ -1039,33 +1084,6 @@ sons: intVal: 54 postfix: - "# discard value" - - kind: "nkBlockStmt" - sons: - - kind: "nkEmpty" - - kind: "nkStmtList" - sons: - - kind: "nkCall" - sons: - - kind: "nkDotExpr" - prefix: - - "# also after call" - sons: - - kind: "nkCall" - sons: - - kind: "nkDotExpr" - prefix: - - "# comment between the dots" - sons: - - kind: "nkDotExpr" - sons: - - kind: "nkIdent" - ident: "f" - - kind: "nkIdent" - ident: "x" - - kind: "nkIdent" - ident: "z" - - kind: "nkIdent" - ident: "d" - kind: "nkProcDef" sons: - kind: "nkIdent" @@ -1079,10 +1097,11 @@ sons: - kind: "nkEmpty" - kind: "nkEmpty" - kind: "nkStmtList" - prefix: - - "## Comment here" - - "## another" sons: + - kind: "nkCommentStmt" + "comment": "## Comment here" + - kind: "nkCommentStmt" + "comment": "## another" - kind: "nkPar" sons: - kind: "nkInfix" @@ -1106,10 +1125,11 @@ sons: - kind: "nkEmpty" - kind: "nkEmpty" - kind: "nkStmtList" - prefix: - - "## Comment here" - - "## another" sons: + - kind: "nkCommentStmt" + "comment": "## Comment here" + - kind: "nkCommentStmt" + "comment": "## another" - kind: "nkIfStmt" sons: - kind: "nkElifBranch" @@ -1123,10 +1143,11 @@ sons: - kind: "nkElse" sons: - kind: "nkStmtList" - prefix: - - "## comment" - - "## comment 2" sons: + - kind: "nkCommentStmt" + "comment": "## comment" + - kind: "nkCommentStmt" + "comment": "## comment 2" - kind: "nkIfStmt" sons: - kind: "nkElifBranch" @@ -1134,10 +1155,11 @@ sons: - kind: "nkIdent" ident: "true" - kind: "nkStmtList" - prefix: - - "## comment" - - "## comment 2" sons: + - kind: "nkCommentStmt" + "comment": "## comment" + - kind: "nkCommentStmt" + "comment": "## comment 2" - kind: "nkPar" sons: - kind: "nkInfix" @@ -1181,4 +1203,164 @@ sons: - kind: "nkStrLit" strVal: "third arg" postfix: - - "# third arg comment" \ No newline at end of file + - "# third arg comment" + - kind: "nkCommand" + sons: + - kind: "nkIdent" + ident: "command" + - kind: "nkStrLit" + strVal: "first arg" + - kind: "nkCommentStmt" + "comment": "# comment after command" + - kind: "nkCommand" + sons: + - kind: "nkIdent" + ident: "command" + - kind: "nkIntLit" + intVal: 234 + postfix: + - "# command after ind" + - kind: "nkWhenStmt" + sons: + - kind: "nkElifBranch" + sons: + - kind: "nkIdent" + ident: "false" + - kind: "nkStmtList" + sons: + - kind: "nkCommand" + sons: + - kind: "nkIdent" + ident: "command" + - kind: "nkStrLit" + strVal: "first arg" + - kind: "nkCommentStmt" + "comment": "# comment after command nest" + - kind: "nkWhenStmt" + sons: + - kind: "nkElifBranch" + sons: + - kind: "nkIdent" + ident: "false" + - kind: "nkStmtList" + sons: + - kind: "nkCommand" + sons: + - kind: "nkIdent" + ident: "command" + - kind: "nkDotExpr" + sons: + - kind: "nkIdent" + ident: "a" + - kind: "nkIdent" + ident: "b" + - kind: "nkCommentStmt" + "comment": "# comment after command dotexpr nest" + - kind: "nkCall" + sons: + - kind: "nkIdent" + ident: "call" + postfix: + - "# call eol" + - kind: "nkCommand" + sons: + - kind: "nkIdent" + ident: "echo" + - kind: "nkDotExpr" + sons: + - kind: "nkIdent" + ident: "dotexpr" + - kind: "nkIdent" + ident: "dot" + postfix: + - "# after dotexpr in command" + - kind: "nkCommentStmt" + "comment": "# between two dotepxrs" + - kind: "nkDotExpr" + mid: + - "# between dotexpr and dotonnewline" + sons: + - kind: "nkIdent" + ident: "dotexpr" + - kind: "nkIdent" + ident: "dotonnewline" + - kind: "nkIfStmt" + sons: + - kind: "nkElifBranch" + sons: + - kind: "nkIdent" + ident: "true" + - kind: "nkStmtList" + sons: + - kind: "nkCommand" + sons: + - kind: "nkIdent" + ident: "echo" + - kind: "nkDotExpr" + sons: + - kind: "nkIdent" + ident: "dotexpr" + - kind: "nkIdent" + ident: "dot" + postfix: + - "# after dotexpr in command ind" + - kind: "nkCommentStmt" + "comment": "# between two dotepxrs ind" + - kind: "nkDotExpr" + mid: + - "# between dotexpr and dotonnewline ind" + sons: + - kind: "nkIdent" + ident: "dotexpr" + - kind: "nkIdent" + ident: "dotonnewline" + - kind: "nkBlockStmt" + sons: + - kind: "nkEmpty" + - kind: "nkStmtList" + sons: + - kind: "nkCall" + sons: + - kind: "nkDotExpr" + mid: + - "# also after call" + sons: + - kind: "nkCall" + sons: + - kind: "nkDotExpr" + mid: + - "# comment between the dots" + sons: + - kind: "nkDotExpr" + sons: + - kind: "nkIdent" + ident: "f" + - kind: "nkIdent" + ident: "x" + - kind: "nkIdent" + ident: "z" + - kind: "nkIdent" + ident: "d" + postfix: + - "# far eol of dotexpr" + - kind: "nkCommentStmt" + "comment": "# after dotexpr ind" + - kind: "nkBlockStmt" + sons: + - kind: "nkEmpty" + - kind: "nkStmtList" + sons: + - kind: "nkCommentStmt" + "comment": "# no whitespace between the next two multilines" + - kind: "nkCall" + sons: + - kind: "nkIdent" + ident: "functionCall" + - kind: "nkIdent" + ident: "param" + - kind: "nkCall" + sons: + - kind: "nkIdent" + ident: "functionCall" + - kind: "nkIdent" + ident: "param" \ No newline at end of file diff --git a/tests/before/exprs.nim b/tests/before/exprs.nim index dc6bba5..49afcdd 100644 --- a/tests/before/exprs.nim +++ b/tests/before/exprs.nim @@ -24,6 +24,8 @@ template ttt*: untyped = discard xxxx "arg": yyy +# TODO +# (try: (discard rOk.tryError(); false) except ResultError[int]: true) res.add ( fff() do: @@ -61,4 +63,5 @@ of aaaa: discard of aaaaaaaaaaaaaaaaaaaaaaaaaaaaa, bbbbbbbbbbbbbbbbbbbbbbbbbb, ccccccccccccccccccccccc, dddddddddddddddddddddd, aaaaaaaaaaaaaaaaaaaaaaaaaaaaa, bbbbbbbbbbbbbbbbbbbbbbbbbb, ccccccccccccccccccccccc, dddddddddddddddddddddd: discard else: - discard \ No newline at end of file + discard + diff --git a/tests/before/exprs.nim.nph.yaml b/tests/before/exprs.nim.nph.yaml index c8f8bdd..cca62e6 100644 --- a/tests/before/exprs.nim.nph.yaml +++ b/tests/before/exprs.nim.nph.yaml @@ -112,9 +112,9 @@ sons: - kind: "nkDiscardStmt" sons: - kind: "nkEmpty" + - kind: "nkCommentStmt" + "comment": "# needs spaces" - kind: "nkForStmt" - prefix: - - "# needs spaces" sons: - kind: "nkIdent" ident: "a" @@ -168,9 +168,9 @@ sons: sons: - kind: "nkIdent" ident: "xxx" + - kind: "nkCommentStmt" + "comment": "# command syntax with colon" - kind: "nkDiscardStmt" - prefix: - - "# command syntax with colon" sons: - kind: "nkCommand" sons: @@ -182,6 +182,10 @@ sons: sons: - kind: "nkIdent" ident: "yyy" + - kind: "nkCommentStmt" + "comment": "# TODO" + - kind: "nkCommentStmt" + "comment": "# (try: (discard rOk.tryError(); false) except ResultError[int]: true)" - kind: "nkCommand" sons: - kind: "nkDotExpr" @@ -200,11 +204,13 @@ sons: sons: - kind: "nkIdent" ident: "yy" + - kind: "nkCommentStmt" + "comment": "# we don\'t what `do` in let but need it in the above commend - this needs" + - kind: "nkCommentStmt" + "comment": "# more investigation - this is important for templates calls like Result.valueOr" + - kind: "nkCommentStmt" + "comment": "# which become ugly otherwise" - kind: "nkLetSection" - prefix: - - "# we don\'t what `do` in let but need it in the above commend - this needs" - - "# more investigation - this is important for templates calls like Result.valueOr" - - "# which become ugly otherwise" sons: - kind: "nkIdentDefs" sons: