From de0938e7b39455f8432aed45f87fefe16b3d563e Mon Sep 17 00:00:00 2001 From: Mikhail Yakshin Date: Fri, 24 Mar 2017 05:37:10 +0300 Subject: [PATCH 01/90] Added --read-write switch --- jvm/src/main/scala/io/kaitai/struct/JavaMain.scala | 4 ++++ shared/src/main/scala/io/kaitai/struct/RuntimeConfig.scala | 1 + 2 files changed, 5 insertions(+) diff --git a/jvm/src/main/scala/io/kaitai/struct/JavaMain.scala b/jvm/src/main/scala/io/kaitai/struct/JavaMain.scala index e66307dae..83d791e36 100644 --- a/jvm/src/main/scala/io/kaitai/struct/JavaMain.scala +++ b/jvm/src/main/scala/io/kaitai/struct/JavaMain.scala @@ -54,6 +54,10 @@ object JavaMain { } } + opt[Unit]('w', "read-write") action { (x, c) => + c.copy(runtime = c.runtime.copy(readWrite = true)) + } text("generate read-write support in classes (default: read-only)") + opt[File]('d', "outdir") valueName("") action { (x, c) => c.copy(outDir = x) } text("output directory (filenames will be auto-generated)") diff --git a/shared/src/main/scala/io/kaitai/struct/RuntimeConfig.scala b/shared/src/main/scala/io/kaitai/struct/RuntimeConfig.scala index 5a782c02d..42bb37501 100644 --- a/shared/src/main/scala/io/kaitai/struct/RuntimeConfig.scala +++ b/shared/src/main/scala/io/kaitai/struct/RuntimeConfig.scala @@ -3,6 +3,7 @@ package io.kaitai.struct case class RuntimeConfig( debug: Boolean = false, opaqueTypes: Boolean = false, + readWrite: Boolean = false, javaPackage: String = "", dotNetNamespace: String = "Kaitai", phpNamespace: String = "" From d24a36dfc46c6050e6b1e1435710de19b21ef9c8 Mon Sep 17 00:00:00 2001 From: Mikhail Yakshin Date: Fri, 24 Mar 2017 05:53:21 +0300 Subject: [PATCH 02/90] JavaMain: ensure that exceptions are really thrown, even on compile stage --- jvm/src/main/scala/io/kaitai/struct/JavaMain.scala | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/jvm/src/main/scala/io/kaitai/struct/JavaMain.scala b/jvm/src/main/scala/io/kaitai/struct/JavaMain.scala index 83d791e36..b444854b3 100644 --- a/jvm/src/main/scala/io/kaitai/struct/JavaMain.scala +++ b/jvm/src/main/scala/io/kaitai/struct/JavaMain.scala @@ -244,11 +244,15 @@ class JavaMain(config: CLIConfig) { Log.fileOps.info(() => s"... compiling it for $langStr... ") val lang = LanguageCompilerStatic.byString(langStr) specs.map { case (_, classSpec) => - val res = try { + val res = if (config.throwExceptions) { compileSpecAndWriteToFile(classSpec, lang, outDir) - } catch { - case ex: Throwable => - SpecFailure(List(exceptionToCompileError(ex, classSpec.nameAsStr))) + } else { + try { + compileSpecAndWriteToFile(classSpec, lang, outDir) + } catch { + case ex: Throwable => + SpecFailure(List(exceptionToCompileError(ex, classSpec.nameAsStr))) + } } classSpec.nameAsStr -> res }.toMap From 0b757b9c85677e046f57f338a087d57d831031f7 Mon Sep 17 00:00:00 2001 From: Mikhail Yakshin Date: Fri, 24 Mar 2017 10:07:20 +0300 Subject: [PATCH 03/90] Started PoC seq writer support in JavaCompiler --- .../io/kaitai/struct/ClassCompiler.scala | 21 ++++- .../struct/languages/JavaCompiler.scala | 58 ++++++++++++-- .../components/EveryWriteIsExpression.scala | 77 +++++++++++++++++++ .../components/LanguageCompiler.scala | 4 + .../components/UniversalFooter.scala | 4 +- 5 files changed, 156 insertions(+), 8 deletions(-) create mode 100644 shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala diff --git a/shared/src/main/scala/io/kaitai/struct/ClassCompiler.scala b/shared/src/main/scala/io/kaitai/struct/ClassCompiler.scala index b15b468fc..3327457f8 100644 --- a/shared/src/main/scala/io/kaitai/struct/ClassCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/ClassCompiler.scala @@ -58,9 +58,15 @@ class ClassCompiler( lang.classConstructorHeader(curClass.name, curClass.parentTypeName, topClassName) curClass.instances.foreach { case (instName, _) => lang.instanceClear(instName) } - compileSeq(curClass.seq, extraAttrs) + compileSeqRead(curClass.seq, extraAttrs) lang.classConstructorFooter + if (config.readWrite) { + lang.funcWriteHeader(curClass) + compileSeqWrite(curClass.seq, extraAttrs) + lang.funcWriteFooter(curClass) + } + lang.classDestructorHeader(curClass.name, curClass.parentTypeName, topClassName) curClass.seq.foreach((attr) => lang.attrDestructor(attr, attr.id)) curClass.instances.foreach { case (id, instSpec) => @@ -97,7 +103,7 @@ class ClassCompiler( compileEnums(curClass) } - def compileSeq(seq: List[AttrSpec], extraAttrs: ListBuffer[AttrSpec]) = { + def compileSeqRead(seq: List[AttrSpec], extraAttrs: ListBuffer[AttrSpec]) = { var wasUnaligned = false seq.foreach { (attr) => val nowUnaligned = isUnalignedBits(attr.dataType) @@ -108,6 +114,17 @@ class ClassCompiler( } } + def compileSeqWrite(seq: List[AttrSpec], extraAttrs: ListBuffer[AttrSpec]) = { + var wasUnaligned = false + seq.foreach { (attr) => + val nowUnaligned = isUnalignedBits(attr.dataType) + if (wasUnaligned && !nowUnaligned) + lang.alignToByte(lang.normalIO) + lang.attrWrite(attr, attr.id, extraAttrs) + wasUnaligned = nowUnaligned + } + } + def compileEnums(curClass: ClassSpec): Unit = curClass.enums.foreach { case(_, enumColl) => compileEnum(curClass, enumColl) } diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala index 503a9c206..3e5fb0664 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala @@ -15,6 +15,7 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) with UpperCamelCaseClasses with ObjectOrientedLanguage with EveryReadIsExpression + with EveryWriteIsExpression with UniversalFooter with UniversalDoc with AllocateIOLocalVar @@ -85,7 +86,7 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts("super(_io);") if (name == rootClassName) out.puts("this._root = this;") - if (!debug) + if (!(debug || config.readWrite)) out.puts("_read();") out.dec out.puts("}") @@ -97,7 +98,7 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts("this._parent = _parent;") if (name == rootClassName) out.puts("this._root = this;") - if (!debug) + if (!(debug || config.readWrite)) out.puts("_read();") out.dec out.puts("}") @@ -108,12 +109,13 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts("super(_io);") out.puts("this._parent = _parent;") out.puts("this._root = _root;") - if (!debug) + if (!(debug || config.readWrite)) out.puts("_read();") out.dec out.puts("}") - val readAccessAndType = if (debug) { + out.puts + val readAccessAndType = if (debug || config.readWrite) { "public" } else { "private" @@ -131,7 +133,14 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } override def attributeReader(attrName: Identifier, attrType: DataType, condSpec: ConditionalSpec): Unit = { - out.puts(s"public ${kaitaiType2JavaType(attrType, condSpec)} ${idToStr(attrName)}() { return ${idToStr(attrName)}; }") + val javaType = kaitaiType2JavaType(attrType, condSpec) + val name = idToStr(attrName) + + out.puts(s"public $javaType $name() { return $name; }") + + if (config.readWrite) { + out.puts(s"public void set${idToSetterStr(attrName)}($javaType _v) { $name = _v; }") + } } override def universalDoc(doc: DocSpec): Unit = { @@ -152,6 +161,11 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts( " */") } + override def funcWriteHeader(curClass: ClassSpec): Unit = { + out.puts("public void _write() {") + out.inc + } + override def attrFixedContentsParse(attrName: Identifier, contents: String): Unit = { out.puts(s"${privateMemberName(attrName)} = $normalIO.ensureFixedContents($contents);") } @@ -468,6 +482,30 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts(s"public static String[] _seqFields = new String[] { $seqStr };") } + override def attrPrimitiveWrite(io: String, expr: String, dataType: DataType): Unit = { + val stmt = dataType match { + case t: ReadableType => + s"$io.write${Utils.capitalize(t.apiCall)}($expr)" + case BitsType1 => + s"$io.writeBitsInt(1, ($expr) ? 1 : 0)" + case BitsType(width: Int) => + s"$io.writeBitsInt($width, $expr)" + case t: UserType => + val addArgs = if (t.isOpaque) { + "" + } else { + val parent = t.forcedParent match { + case Some(USER_TYPE_NO_PARENT) => "null" + case Some(fp) => translator.translate(fp) + case None => "this" + } + s", $parent, _root" + } + s"new ${types2class(t.name)}($io$addArgs)" + } + out.puts(stmt + ";") + } + def value2Const(s: String) = s.toUpperCase def idToStr(id: Identifier): String = { @@ -480,6 +518,16 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } } + def idToSetterStr(id: Identifier): String = { + id match { + case SpecialIdentifier(name) => name + case NamedIdentifier(name) => Utils.upperCamelCase(name) + case NumberedIdentifier(idx) => s"_${NumberedIdentifier.TEMPLATE}$idx" + case InstanceIdentifier(name) => Utils.upperCamelCase(name) + case RawIdentifier(innerId) => "_raw_" + idToSetterStr(innerId) + } + } + override def privateMemberName(id: Identifier): String = s"this.${idToStr(id)}" override def publicMemberName(id: Identifier) = idToStr(id) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala new file mode 100644 index 000000000..e9e0d5dda --- /dev/null +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala @@ -0,0 +1,77 @@ +package io.kaitai.struct.languages.components + +import io.kaitai.struct.datatype.DataType +import io.kaitai.struct.datatype.DataType._ +import io.kaitai.struct.exprlang.Ast +import io.kaitai.struct.format._ + +import scala.collection.mutable.ListBuffer + +trait EveryWriteIsExpression extends LanguageCompiler with ObjectOrientedLanguage { + override def attrWrite(attr: AttrLikeSpec, id: Identifier, extraAttrs: ListBuffer[AttrSpec]): Unit = { + attrParseIfHeader(id, attr.cond.ifExpr) + + val io = normalIO + + attr.cond.repeat match { + case RepeatEos => + condRepeatEosHeader(id, io, attr.dataType, needRaww(attr.dataType)) + attrWrite2(id, attr.dataType, io, extraAttrs, attr.cond.repeat, false) + condRepeatEosFooter + case RepeatExpr(repeatExpr: Ast.expr) => + condRepeatExprHeader(id, io, attr.dataType, needRaww(attr.dataType), repeatExpr) + attrWrite2(id, attr.dataType, io, extraAttrs, attr.cond.repeat, false) + condRepeatExprFooter + case RepeatUntil(untilExpr: Ast.expr) => + condRepeatUntilHeader(id, io, attr.dataType, needRaww(attr.dataType), untilExpr) + attrWrite2(id, attr.dataType, io, extraAttrs, attr.cond.repeat, false) + condRepeatUntilFooter(id, io, attr.dataType, needRaww(attr.dataType), untilExpr) + case NoRepeat => + attrWrite2(id, attr.dataType, io, extraAttrs, attr.cond.repeat, false) + } + + attrParseIfFooter(attr.cond.ifExpr) + } + + def attrWrite2( + id: Identifier, + dataType: DataType, + io: String, + extraAttrs: ListBuffer[AttrSpec], + rep: RepeatSpec, + isRaw: Boolean + ): Unit = { + dataType match { +// case FixedBytesType(c, _) => +// attrBytesTypeWrite(id, c) +// case t: UserType => +// attrUserTypeWrite(id, t, io, extraAttrs, rep) +// case t: BytesType => +// attrBytesTypeWrite(id, t, io, extraAttrs, rep, isRaw) +// case SwitchType(on, cases) => +// attrSwitchTypeWrite(id, on, cases, io, extraAttrs, rep) +// case t: StrFromBytesType => +// val expr = translator.bytesToStr(parseExprBytes(t.bytes, io), Ast.expr.Str(t.encoding)) +// handleAssignment(id, expr, rep, isRaw) + case t: EnumType => + val expr = translator.enumToInt(Ast.expr.Name(Ast.identifier(idToStr(id))), t) + attrPrimitiveWrite(io, expr, t.basedOn) + case _ => + val expr = writeExpr(id, rep, isRaw) + attrPrimitiveWrite(io, expr, dataType) + } + } + + def writeExpr(id: Identifier, rep: RepeatSpec, isRaw: Boolean): String = + privateMemberName(id) + + def attrPrimitiveWrite(io: String, expr: String, dt: DataType): Unit + + // FIXME: unify with EveryReadIsExpression + def needRaww(dataType: DataType): Boolean = { + dataType match { + case t: UserTypeFromBytes => true + case _ => false + } + } +} diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala index c2dff3f6a..74780f1a3 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala @@ -76,6 +76,10 @@ abstract class LanguageCompiler( def attrParse(attr: AttrLikeSpec, id: Identifier, extraAttrs: ListBuffer[AttrSpec]): Unit def attrDestructor(attr: AttrLikeSpec, id: Identifier): Unit = {} + def funcWriteHeader(curClass: ClassSpec): Unit = ??? + def funcWriteFooter(curClass: ClassSpec): Unit = ??? + def attrWrite(attr: AttrLikeSpec, id: Identifier, extraAttrs: ListBuffer[AttrSpec]): Unit = ??? + def attrFixedContentsParse(attrName: Identifier, contents: Array[Byte]): Unit def condIfSetNull(instName: Identifier): Unit = {} diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/UniversalFooter.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/UniversalFooter.scala index bbfd392b8..9d2d13e78 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/UniversalFooter.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/UniversalFooter.scala @@ -1,12 +1,13 @@ package io.kaitai.struct.languages.components import io.kaitai.struct.exprlang.Ast.expr +import io.kaitai.struct.format.ClassSpec /** * All footers in the language look the same and can be written by the same * simple argument-less method. */ -trait UniversalFooter { +trait UniversalFooter extends LanguageCompiler { /** * Single method that outputs all kind of footers in the language. */ @@ -14,6 +15,7 @@ trait UniversalFooter { def classFooter(name: String): Unit = universalFooter def classConstructorFooter: Unit = universalFooter + override def funcWriteFooter(curClass: ClassSpec): Unit = universalFooter def condRepeatExprFooter = universalFooter def condRepeatEosFooter: Unit = universalFooter def condIfFooter(expr: expr): Unit = universalFooter From 93b37a135de56fb5995ba79fbff3be28efe2a2a5 Mon Sep 17 00:00:00 2001 From: Mikhail Yakshin Date: Fri, 24 Mar 2017 10:43:29 +0300 Subject: [PATCH 04/90] JavaCompiler: very basic bytes writing support --- .../kaitai/struct/languages/JavaCompiler.scala | 2 ++ .../components/EveryWriteIsExpression.scala | 17 +++++++++++++++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala index 3e5fb0664..a7a44c601 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala @@ -490,6 +490,8 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) s"$io.writeBitsInt(1, ($expr) ? 1 : 0)" case BitsType(width: Int) => s"$io.writeBitsInt($width, $expr)" + case _: BytesType => + s"$io.writeBytes($expr)" case t: UserType => val addArgs = if (t.isOpaque) { "" diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala index e9e0d5dda..cbaf8f9a5 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala @@ -46,8 +46,8 @@ trait EveryWriteIsExpression extends LanguageCompiler with ObjectOrientedLanguag // attrBytesTypeWrite(id, c) // case t: UserType => // attrUserTypeWrite(id, t, io, extraAttrs, rep) -// case t: BytesType => -// attrBytesTypeWrite(id, t, io, extraAttrs, rep, isRaw) + case t: BytesType => + attrBytesTypeWrite(id, t, io, extraAttrs, rep, isRaw) // case SwitchType(on, cases) => // attrSwitchTypeWrite(id, on, cases, io, extraAttrs, rep) // case t: StrFromBytesType => @@ -65,6 +65,19 @@ trait EveryWriteIsExpression extends LanguageCompiler with ObjectOrientedLanguag def writeExpr(id: Identifier, rep: RepeatSpec, isRaw: Boolean): String = privateMemberName(id) + def attrBytesTypeWrite(id: Identifier, t: BytesType, io: String, extraAttrs: ListBuffer[AttrSpec], rep: RepeatSpec, isRaw: Boolean) = { + t match { +// case FixedBytesType(contents, process) => +// case BytesEosType(terminator, include, padRight, process) => +// case BytesLimitType(size, terminator, include, padRight, process) => + case t: BytesTerminatedType => + val expr = writeExpr(id, rep, isRaw) + attrPrimitiveWrite(io, expr, t) + if (t.consume && !t.include) + attrPrimitiveWrite(io, t.terminator.toString, Int1Type(false)) + } + } + def attrPrimitiveWrite(io: String, expr: String, dt: DataType): Unit // FIXME: unify with EveryReadIsExpression From 75ec0c26343365ab85fad20d62b34c0739102768 Mon Sep 17 00:00:00 2001 From: Mikhail Yakshin Date: Fri, 24 Mar 2017 11:41:15 +0300 Subject: [PATCH 05/90] Allow simple serialization of FixedBytesType --- .../struct/languages/components/EveryWriteIsExpression.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala index cbaf8f9a5..6d6762029 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala @@ -67,7 +67,8 @@ trait EveryWriteIsExpression extends LanguageCompiler with ObjectOrientedLanguag def attrBytesTypeWrite(id: Identifier, t: BytesType, io: String, extraAttrs: ListBuffer[AttrSpec], rep: RepeatSpec, isRaw: Boolean) = { t match { -// case FixedBytesType(contents, process) => + case FixedBytesType(contents, process) => + attrPrimitiveWrite(io, translator.doByteArrayLiteral(contents), t) // case BytesEosType(terminator, include, padRight, process) => // case BytesLimitType(size, terminator, include, padRight, process) => case t: BytesTerminatedType => From eed5d0769d5dd2c25fc1b2e4cc4bd800e3c57388 Mon Sep 17 00:00:00 2001 From: Mikhail Yakshin Date: Fri, 24 Mar 2017 14:14:09 +0300 Subject: [PATCH 06/90] Translators: added strToBytes, implemented in Java translator --- .../scala/io/kaitai/struct/translators/BaseTranslator.scala | 1 + .../scala/io/kaitai/struct/translators/JavaTranslator.scala | 2 ++ 2 files changed, 3 insertions(+) diff --git a/shared/src/main/scala/io/kaitai/struct/translators/BaseTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/BaseTranslator.scala index 73e411f10..4f4aa793c 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/BaseTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/BaseTranslator.scala @@ -296,6 +296,7 @@ abstract class BaseTranslator(val provider: TypeProvider) extends TypeDetector(p doIfExp(value, Ast.expr.IntNum(1), Ast.expr.IntNum(0)) def intToStr(i: Ast.expr, base: Ast.expr): String def bytesToStr(bytesExpr: String, encoding: Ast.expr): String + def strToBytes(strExpr: String, encoding: Ast.expr): String = ??? def strLength(s: Ast.expr): String def strReverse(s: Ast.expr): String def strSubstring(s: Ast.expr, from: Ast.expr, to: Ast.expr): String diff --git a/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala index 81e4a3057..56e192d87 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala @@ -80,6 +80,8 @@ class JavaTranslator(provider: TypeProvider) extends BaseTranslator(provider) { s"Long.toString(${translate(i)}, ${translate(base)})" override def bytesToStr(bytesExpr: String, encoding: Ast.expr): String = s"new String($bytesExpr, Charset.forName(${translate(encoding)}))" + override def strToBytes(strExpr: String, encoding: expr): String = + s"($strExpr).getBytes(Charset.forName(${translate(encoding)}))" override def strLength(s: expr): String = s"${translate(s)}.length()" override def strReverse(s: expr): String = From 5cf4edd3d01e7cd34bd3d95f170a4c3a2110cb5e Mon Sep 17 00:00:00 2001 From: Mikhail Yakshin Date: Fri, 24 Mar 2017 14:55:53 +0300 Subject: [PATCH 07/90] Added _i iteration number identifier --- shared/src/main/scala/io/kaitai/struct/format/Identifier.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/shared/src/main/scala/io/kaitai/struct/format/Identifier.scala b/shared/src/main/scala/io/kaitai/struct/format/Identifier.scala index 01d6fbaa6..f907de756 100644 --- a/shared/src/main/scala/io/kaitai/struct/format/Identifier.scala +++ b/shared/src/main/scala/io/kaitai/struct/format/Identifier.scala @@ -60,6 +60,7 @@ object Identifier { val IO = "_io" val ITERATOR = "_" val ITERATOR2 = "_buf" + val ITERATOR_I = "_i" } case class RawIdentifier(innerId: Identifier) extends Identifier From 4a3535f9e000ebf92b2cbbbaf13ab0011a4694f7 Mon Sep 17 00:00:00 2001 From: Mikhail Yakshin Date: Fri, 24 Mar 2017 14:56:13 +0300 Subject: [PATCH 08/90] JavaTranslator: added translation of `_i` --- .../main/scala/io/kaitai/struct/translators/JavaTranslator.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala index 56e192d87..358817db6 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala @@ -41,6 +41,7 @@ class JavaTranslator(provider: TypeProvider) extends BaseTranslator(provider) { case Identifier.IO => "_io()" case Identifier.ITERATOR => "_it" case Identifier.ITERATOR2 => "_buf" + case Identifier.ITERATOR_I => "_i" case _ => s"${Utils.lowerCamelCase(s)}()" } From 568964dcdd85cedc4374b3688c95e3dd3cee0f8a Mon Sep 17 00:00:00 2001 From: Mikhail Yakshin Date: Fri, 24 Mar 2017 14:57:44 +0300 Subject: [PATCH 09/90] ClassTypeProvider: use constants + added ITERATOR_I handling --- .../scala/io/kaitai/struct/ClassTypeProvider.scala | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/ClassTypeProvider.scala b/shared/src/main/scala/io/kaitai/struct/ClassTypeProvider.scala index be6172e2f..6f443e920 100644 --- a/shared/src/main/scala/io/kaitai/struct/ClassTypeProvider.scala +++ b/shared/src/main/scala/io/kaitai/struct/ClassTypeProvider.scala @@ -20,16 +20,18 @@ class ClassTypeProvider(topClass: ClassSpec) extends TypeProvider { override def determineType(inClass: ClassSpec, attrName: String): DataType = { attrName match { - case "_root" => + case Identifier.ROOT => makeUserType(topClass) - case "_parent" => + case Identifier.PARENT => if (inClass.parentClass == UnknownClassSpec) throw new RuntimeException(s"Unable to derive _parent type in ${inClass.name.mkString("::")}") makeUserType(inClass.parentClass) - case "_io" => + case Identifier.IO => KaitaiStreamType - case "_" => + case Identifier.ITERATOR => currentIteratorType + case Identifier.ITERATOR_I => + CalcIntType case "_on" => currentSwitchType case _ => From 9e7bffa42db6b2ebf6ebb2142126e0a84474554d Mon Sep 17 00:00:00 2001 From: Mikhail Yakshin Date: Fri, 24 Mar 2017 15:16:26 +0300 Subject: [PATCH 10/90] Writing: added string handling, some repeat-expr handling, BytesEos handling --- .../struct/languages/JavaCompiler.scala | 6 ++- .../components/EveryWriteIsExpression.scala | 52 +++++++++++++++---- 2 files changed, 48 insertions(+), 10 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala index a7a44c601..ecf1eccf7 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala @@ -285,7 +285,11 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) if (needRaw) out.puts(s"${privateMemberName(RawIdentifier(id))} = new ArrayList((int) (${expression(repeatExpr)}));") out.puts(s"${idToStr(id)} = new ${kaitaiType2JavaType(ArrayType(dataType))}((int) (${expression(repeatExpr)}));") - out.puts(s"for (int i = 0; i < ${expression(repeatExpr)}; i++) {") + condRepeatExprHeader2(id, io, dataType, needRaw, repeatExpr) + } + + override def condRepeatExprHeader2(id: Identifier, io: String, dataType: DataType, needRaw: Boolean, repeatExpr: expr): Unit = { + out.puts(s"for (int _i = 0; _i < ${expression(repeatExpr)}; _i++) {") out.inc } diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala index 6d6762029..8f8b51507 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala @@ -19,7 +19,7 @@ trait EveryWriteIsExpression extends LanguageCompiler with ObjectOrientedLanguag attrWrite2(id, attr.dataType, io, extraAttrs, attr.cond.repeat, false) condRepeatEosFooter case RepeatExpr(repeatExpr: Ast.expr) => - condRepeatExprHeader(id, io, attr.dataType, needRaww(attr.dataType), repeatExpr) + condRepeatExprHeader2(id, io, attr.dataType, needRaww(attr.dataType), repeatExpr) attrWrite2(id, attr.dataType, io, extraAttrs, attr.cond.repeat, false) condRepeatExprFooter case RepeatUntil(untilExpr: Ast.expr) => @@ -42,17 +42,14 @@ trait EveryWriteIsExpression extends LanguageCompiler with ObjectOrientedLanguag isRaw: Boolean ): Unit = { dataType match { -// case FixedBytesType(c, _) => -// attrBytesTypeWrite(id, c) // case t: UserType => // attrUserTypeWrite(id, t, io, extraAttrs, rep) case t: BytesType => attrBytesTypeWrite(id, t, io, extraAttrs, rep, isRaw) // case SwitchType(on, cases) => // attrSwitchTypeWrite(id, on, cases, io, extraAttrs, rep) -// case t: StrFromBytesType => -// val expr = translator.bytesToStr(parseExprBytes(t.bytes, io), Ast.expr.Str(t.encoding)) -// handleAssignment(id, expr, rep, isRaw) + case t: StrFromBytesType => + attrStrTypeWrite(id, t, io, extraAttrs, rep, isRaw) case t: EnumType => val expr = translator.enumToInt(Ast.expr.Name(Ast.identifier(idToStr(id))), t) attrPrimitiveWrite(io, expr, t.basedOn) @@ -62,14 +59,29 @@ trait EveryWriteIsExpression extends LanguageCompiler with ObjectOrientedLanguag } } - def writeExpr(id: Identifier, rep: RepeatSpec, isRaw: Boolean): String = - privateMemberName(id) + def writeExpr(id: Identifier, rep: RepeatSpec, isRaw: Boolean): String = { + rep match { + case _: RepeatExpr => + translator.translate( + Ast.expr.Subscript( + Ast.expr.Name(Ast.identifier(idToStr(id))), + Ast.expr.Name(Ast.identifier(Identifier.ITERATOR_I)) + ) + ) + case NoRepeat => + privateMemberName(id) + } + } def attrBytesTypeWrite(id: Identifier, t: BytesType, io: String, extraAttrs: ListBuffer[AttrSpec], rep: RepeatSpec, isRaw: Boolean) = { t match { case FixedBytesType(contents, process) => attrPrimitiveWrite(io, translator.doByteArrayLiteral(contents), t) -// case BytesEosType(terminator, include, padRight, process) => + case t: BytesEosType => + val expr = writeExpr(id, rep, isRaw) + attrPrimitiveWrite(io, expr, t) + if (t.terminator.isDefined && !t.include) + attrPrimitiveWrite(io, t.terminator.toString, Int1Type(false)) // case BytesLimitType(size, terminator, include, padRight, process) => case t: BytesTerminatedType => val expr = writeExpr(id, rep, isRaw) @@ -79,6 +91,26 @@ trait EveryWriteIsExpression extends LanguageCompiler with ObjectOrientedLanguag } } + def attrStrTypeWrite(id: Identifier, t: StrFromBytesType, io: String, extraAttrs: ListBuffer[AttrSpec], rep: RepeatSpec, isRaw: Boolean) = { + val expr = translator.strToBytes(writeExpr(id, rep, isRaw), Ast.expr.Str(t.encoding)) + attrPrimitiveWrite(io, expr, t.bytes) + + t.bytes match { + case t: BytesEosType => + if (t.terminator.isDefined && !t.include) + attrPrimitiveWrite(io, t.terminator.toString, Int1Type(false)) + case t: BytesLimitType => + // FIXME: implement padding and terminator byte + t.terminator.foreach((terminator) => + if (!t.include) + attrPrimitiveWrite(io, terminator.toString, Int1Type(false)) + ) + case t: BytesTerminatedType => + if (t.consume && !t.include) + attrPrimitiveWrite(io, t.terminator.toString, Int1Type(false)) + } + } + def attrPrimitiveWrite(io: String, expr: String, dt: DataType): Unit // FIXME: unify with EveryReadIsExpression @@ -88,4 +120,6 @@ trait EveryWriteIsExpression extends LanguageCompiler with ObjectOrientedLanguag case _ => false } } + + def condRepeatExprHeader2(id: Identifier, io: String, dataType: DataType, needRaw: Boolean, repeatExpr: Ast.expr): Unit = ??? } From 8fd100291c72689bf0b2f3f1210064c074f4c02a Mon Sep 17 00:00:00 2001 From: Mikhail Yakshin Date: Mon, 27 Mar 2017 16:40:43 +0300 Subject: [PATCH 11/90] Serialization: basic support for user types and BytesLimitType --- .../struct/languages/JavaCompiler.scala | 20 ++++----- .../components/EveryWriteIsExpression.scala | 44 +++++++++++++++++-- 2 files changed, 49 insertions(+), 15 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala index ecf1eccf7..4fd3cb619 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala @@ -9,6 +9,8 @@ import io.kaitai.struct.languages.components._ import io.kaitai.struct.translators.{JavaTranslator, TypeDetector, TypeProvider} import io.kaitai.struct._ +import scala.collection.mutable.ListBuffer + class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) extends LanguageCompiler(typeProvider, config) with SingleOutputFile @@ -496,22 +498,16 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) s"$io.writeBitsInt($width, $expr)" case _: BytesType => s"$io.writeBytes($expr)" - case t: UserType => - val addArgs = if (t.isOpaque) { - "" - } else { - val parent = t.forcedParent match { - case Some(USER_TYPE_NO_PARENT) => "null" - case Some(fp) => translator.translate(fp) - case None => "this" - } - s", $parent, _root" - } - s"new ${types2class(t.name)}($io$addArgs)" } out.puts(stmt + ";") } + override def attrBytesLimitWrite(io: String, expr: String, size: String, term: Int, padRight: Int): Unit = + out.puts(s"$io.writeBytesLimit($expr, $size, (byte) $term, (byte) $padRight);") + + override def attrUserTypeWrite(id: Identifier, t: UserType, io: String, extraAttrs: ListBuffer[AttrSpec], rep: RepeatSpec, isRaw: Boolean): Unit = + out.puts(s"${privateMemberName(id)}._write();") + def value2Const(s: String) = s.toUpperCase def idToStr(id: Identifier): String = { diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala index 8f8b51507..2f1c62093 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala @@ -42,8 +42,8 @@ trait EveryWriteIsExpression extends LanguageCompiler with ObjectOrientedLanguag isRaw: Boolean ): Unit = { dataType match { -// case t: UserType => -// attrUserTypeWrite(id, t, io, extraAttrs, rep) + case t: UserType => + attrUserTypeWrite(id, t, io, extraAttrs, rep, isRaw) case t: BytesType => attrBytesTypeWrite(id, t, io, extraAttrs, rep, isRaw) // case SwitchType(on, cases) => @@ -82,7 +82,9 @@ trait EveryWriteIsExpression extends LanguageCompiler with ObjectOrientedLanguag attrPrimitiveWrite(io, expr, t) if (t.terminator.isDefined && !t.include) attrPrimitiveWrite(io, t.terminator.toString, Int1Type(false)) -// case BytesLimitType(size, terminator, include, padRight, process) => + case blt: BytesLimitType => + val expr = writeExpr(id, rep, isRaw) + attrBytesLimitWrite2(io, expr, blt) case t: BytesTerminatedType => val expr = writeExpr(id, rep, isRaw) attrPrimitiveWrite(io, expr, t) @@ -111,7 +113,43 @@ trait EveryWriteIsExpression extends LanguageCompiler with ObjectOrientedLanguag } } + def attrBytesLimitWrite2(io: String, expr: String, blt: BytesLimitType): Unit = { + val (term, padRight) = (blt.terminator, blt.padRight, blt.include) match { + case (None, None, false) => + // no terminator, no padding => just a regular output + // validation should check that expression's length matches size + attrPrimitiveWrite(io, expr, blt) + return + case (_, None, true) => + // terminator included, no padding => pad with zeroes + (0, 0) + case (_, Some(p), true) => + // terminator included, padding specified + (p, p) + case (Some(t), None, false) => + // only terminator given, don't care about what's gonna go after that + // we'll just pad with zeroes + (t, 0) + case (None, Some(p), false) => + // only padding given, just add terminator equal to padding + (p, p) + case (Some(t), Some(p), false) => + // both terminator and padding specified + (t, p) + } + attrBytesLimitWrite(io, expr, translator.translate(blt.size), term, padRight) + } + def attrPrimitiveWrite(io: String, expr: String, dt: DataType): Unit + def attrBytesLimitWrite(io: String, expr: String, size: String, term: Int, padRight: Int): Unit + def attrUserTypeWrite( + id: Identifier, + t: UserType, + io: String, + extraAttrs: ListBuffer[AttrSpec], + rep: RepeatSpec, + isRaw: Boolean + ): Unit // FIXME: unify with EveryReadIsExpression def needRaww(dataType: DataType): Boolean = { From ea06fec59958e5e10182b328431ddce3fb294f85 Mon Sep 17 00:00:00 2001 From: Mikhail Yakshin Date: Mon, 27 Mar 2017 22:29:36 +0300 Subject: [PATCH 12/90] Serialization: implemented unprocess and slightly better user type writing that gets expression to write properly --- .../struct/languages/JavaCompiler.scala | 23 ++++++++++++++-- .../components/EveryWriteIsExpression.scala | 27 ++++++++++++++----- 2 files changed, 42 insertions(+), 8 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala index 4fd3cb619..03dedcbfd 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala @@ -191,6 +191,25 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } } + override def attrUnprocess(proc: ProcessExpr, varSrc: Identifier, varDest: Identifier): Unit = { + val srcName = privateMemberName(varSrc) + val destName = privateMemberName(varDest) + + proc match { + case ProcessXor(xorValue) => + out.puts(s"$destName = $kstreamName.processXor($srcName, ${expression(xorValue)});") + case ProcessZlib => + out.puts(s"$destName = $kstreamName.unprocessZlib($srcName);") + case ProcessRotate(isLeft, rotValue) => + val expr = if (!isLeft) { + expression(rotValue) + } else { + s"8 - (${expression(rotValue)})" + } + out.puts(s"$destName = $kstreamName.processRotateLeft($srcName, $expr, 1);") + } + } + override def allocateIO(varName: Identifier, rep: RepeatSpec): String = { val javaName = idToStr(varName) @@ -505,8 +524,8 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def attrBytesLimitWrite(io: String, expr: String, size: String, term: Int, padRight: Int): Unit = out.puts(s"$io.writeBytesLimit($expr, $size, (byte) $term, (byte) $padRight);") - override def attrUserTypeWrite(id: Identifier, t: UserType, io: String, extraAttrs: ListBuffer[AttrSpec], rep: RepeatSpec, isRaw: Boolean): Unit = - out.puts(s"${privateMemberName(id)}._write();") + override def attrUserTypeInstreamWrite(expr: String) = + out.puts(s"$expr._write();") def value2Const(s: String) = s.toUpperCase diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala index 2f1c62093..7e747c073 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala @@ -74,19 +74,27 @@ trait EveryWriteIsExpression extends LanguageCompiler with ObjectOrientedLanguag } def attrBytesTypeWrite(id: Identifier, t: BytesType, io: String, extraAttrs: ListBuffer[AttrSpec], rep: RepeatSpec, isRaw: Boolean) = { + val idToWrite = t.process match { + case Some(proc) => + val rawId = RawIdentifier(id) + attrUnprocess(proc, id, rawId) + rawId + case None => + id + } t match { case FixedBytesType(contents, process) => attrPrimitiveWrite(io, translator.doByteArrayLiteral(contents), t) case t: BytesEosType => - val expr = writeExpr(id, rep, isRaw) + val expr = writeExpr(idToWrite, rep, isRaw) attrPrimitiveWrite(io, expr, t) if (t.terminator.isDefined && !t.include) attrPrimitiveWrite(io, t.terminator.toString, Int1Type(false)) case blt: BytesLimitType => - val expr = writeExpr(id, rep, isRaw) + val expr = writeExpr(idToWrite, rep, isRaw) attrBytesLimitWrite2(io, expr, blt) case t: BytesTerminatedType => - val expr = writeExpr(id, rep, isRaw) + val expr = writeExpr(idToWrite, rep, isRaw) attrPrimitiveWrite(io, expr, t) if (t.consume && !t.include) attrPrimitiveWrite(io, t.terminator.toString, Int1Type(false)) @@ -140,8 +148,6 @@ trait EveryWriteIsExpression extends LanguageCompiler with ObjectOrientedLanguag attrBytesLimitWrite(io, expr, translator.translate(blt.size), term, padRight) } - def attrPrimitiveWrite(io: String, expr: String, dt: DataType): Unit - def attrBytesLimitWrite(io: String, expr: String, size: String, term: Int, padRight: Int): Unit def attrUserTypeWrite( id: Identifier, t: UserType, @@ -149,7 +155,16 @@ trait EveryWriteIsExpression extends LanguageCompiler with ObjectOrientedLanguag extraAttrs: ListBuffer[AttrSpec], rep: RepeatSpec, isRaw: Boolean - ): Unit + ) = { + val expr = writeExpr(id, rep, isRaw) + attrUserTypeInstreamWrite(expr) + } + + def attrPrimitiveWrite(io: String, expr: String, dt: DataType): Unit + def attrBytesLimitWrite(io: String, expr: String, size: String, term: Int, padRight: Int): Unit + def attrUserTypeInstreamWrite(expr: String): Unit + + def attrUnprocess(proc: ProcessExpr, varSrc: Identifier, varDest: Identifier): Unit // FIXME: unify with EveryReadIsExpression def needRaww(dataType: DataType): Boolean = { From 9c1793effb5606f16bb3e85301d296d4a75a7a00 Mon Sep 17 00:00:00 2001 From: Mikhail Yakshin Date: Tue, 28 Mar 2017 21:52:18 +0300 Subject: [PATCH 13/90] AllocateIOLocalVar: Added allocation of fixed and growing IOs --- .../components/AllocateIOLocalVar.scala | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/AllocateIOLocalVar.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/AllocateIOLocalVar.scala index 6334d759a..b7b411c03 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/AllocateIOLocalVar.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/AllocateIOLocalVar.scala @@ -8,4 +8,21 @@ import io.kaitai.struct.format.{Identifier, RepeatSpec} */ trait AllocateIOLocalVar { def allocateIO(varName: Identifier, rep: RepeatSpec): String + + /** + * Allocates a fixed-size KaitaiStream IO object for writing into it. + * @param varName variable name to use to generate IO name + * @param size size expression to use + * @return name of generated IO local variable as string + */ + def allocateIOFixed(varName: Identifier, size: String): String = ??? + + /** + * Allocates a growing KaitaiStream IO object for writing into it. + * This one is expected to grow indefinitely as one writes more data + * into it, never raising a buffer overflow exception + * @param varName variable name to use to generate IO name + * @return name of generated IO local variable as string + */ + def allocateIOGrowing(varName: Identifier): String = ??? } From 9f0e331897dbab2de192f2aedc29576c37a70a9d Mon Sep 17 00:00:00 2001 From: Mikhail Yakshin Date: Tue, 28 Mar 2017 21:53:38 +0300 Subject: [PATCH 14/90] Serialization: implemented fixed-size preallocated IO buffers --- .../struct/languages/JavaCompiler.scala | 42 ++++++++++++++++--- .../components/EveryWriteIsExpression.scala | 27 +++++++++++- 2 files changed, 62 insertions(+), 7 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala index 03dedcbfd..23c9632c4 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala @@ -1,5 +1,6 @@ package io.kaitai.struct.languages +import io.kaitai.struct._ import io.kaitai.struct.datatype.DataType import io.kaitai.struct.datatype.DataType._ import io.kaitai.struct.exprlang.Ast @@ -7,9 +8,6 @@ import io.kaitai.struct.exprlang.Ast.expr import io.kaitai.struct.format._ import io.kaitai.struct.languages.components._ import io.kaitai.struct.translators.{JavaTranslator, TypeDetector, TypeProvider} -import io.kaitai.struct._ - -import scala.collection.mutable.ListBuffer class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) extends LanguageCompiler(typeProvider, config) @@ -82,6 +80,14 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } override def classConstructorHeader(name: String, parentClassName: String, rootClassName: String): Unit = { + if (config.readWrite) { + out.puts(s"public ${type2class(name)}() {") + out.inc + out.puts("this(null);") + out.dec + out.puts("}") + } + out.puts out.puts(s"public ${type2class(name)}($kstreamName _io) {") out.inc @@ -164,6 +170,18 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } override def funcWriteHeader(curClass: ClassSpec): Unit = { + out.puts + out.puts(s"public void _write($kstreamName io) {") + out.inc + out.puts("if (this._io == null)") + out.inc + out.puts("this._io = io;") + out.dec + out.puts("_write();") + out.dec + out.puts("}") + + out.puts out.puts("public void _write() {") out.inc } @@ -225,6 +243,17 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) ioName } + override def allocateIOFixed(varName: Identifier, size: String): String = { + val javaName = idToStr(varName) + val ioName = s"_io_$javaName" + + out.puts(s"$kstreamName $ioName = new $kstreamName($size);") + ioName + } + + override def allocateIOGrowing(varName: Identifier): String = + allocateIOFixed(varName, "100000") // FIXME to use real growing buffer + override def useIO(ioEx: expr): String = { out.puts(s"$kstreamName io = ${expression(ioEx)};") "io" @@ -524,8 +553,11 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def attrBytesLimitWrite(io: String, expr: String, size: String, term: Int, padRight: Int): Unit = out.puts(s"$io.writeBytesLimit($expr, $size, (byte) $term, (byte) $padRight);") - override def attrUserTypeInstreamWrite(expr: String) = - out.puts(s"$expr._write();") + override def attrUserTypeInstreamWrite(io: String, expr: String) = + out.puts(s"$expr._write($io);") + + override def attrWriteStreamToStream(srcIo: String, dstIo: String) = + out.puts(s"$dstIo.writeStream($srcIo);") def value2Const(s: String) = s.toUpperCase diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala index 7e747c073..dd8674319 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala @@ -157,12 +157,35 @@ trait EveryWriteIsExpression extends LanguageCompiler with ObjectOrientedLanguag isRaw: Boolean ) = { val expr = writeExpr(id, rep, isRaw) - attrUserTypeInstreamWrite(expr) + + t match { + case _: UserTypeInstream => + attrUserTypeInstreamWrite(io, expr) + case knownSizeType: UserTypeFromBytes => + val rawId = RawIdentifier(id) + val byteType = knownSizeType.bytes + byteType match { + case blt: BytesLimitType => + this match { + // case thisStore: AllocateAndStoreIO => + // val ourIO = thisStore.allocateIO(rawId, rep) + // Utils.addUniqueAttr(extraAttrs, AttrSpec(List(), ourIO, KaitaiStreamType)) + // privateMemberName(ourIO) + case thisLocal: AllocateIOLocalVar => + val ioFixed = thisLocal.allocateIOFixed(rawId, translator.translate(blt.size)) + attrUserTypeInstreamWrite(ioFixed, expr) + attrWriteStreamToStream(ioFixed, io) + } + case _ => + attrUserTypeInstreamWrite(io, expr) + } + } } def attrPrimitiveWrite(io: String, expr: String, dt: DataType): Unit def attrBytesLimitWrite(io: String, expr: String, size: String, term: Int, padRight: Int): Unit - def attrUserTypeInstreamWrite(expr: String): Unit + def attrUserTypeInstreamWrite(io: String, expr: String): Unit + def attrWriteStreamToStream(srcIo: String, dstIo: String): Unit def attrUnprocess(proc: ProcessExpr, varSrc: Identifier, varDest: Identifier): Unit From 969401cd34bf70b645e7203f76ae7abf10ed08eb Mon Sep 17 00:00:00 2001 From: Mikhail Yakshin Date: Tue, 28 Mar 2017 22:29:13 +0300 Subject: [PATCH 15/90] JavaCompiler: generate new Readable/Writable interface implements --- .../io/kaitai/struct/languages/JavaCompiler.scala | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala index 23c9632c4..696b70213 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala @@ -61,7 +61,16 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) "" } - out.puts(s"public ${staticStr}class ${type2class(name)} extends $kstructName {") + val interfaces = if (config.readWrite) { + s"$kstructName.Readable, $kstructName.Writable" + } else { + s"$kstructName.Readable" + } + + out.puts( + s"public ${staticStr}class ${type2class(name)} " + + s"extends $kstructName implements $interfaces {" + ) out.inc if (debug) { From 885469fbeb1c30ee6cf3d9f6d4c760ed3c4b9226 Mon Sep 17 00:00:00 2001 From: Mikhail Yakshin Date: Tue, 28 Mar 2017 22:48:42 +0300 Subject: [PATCH 16/90] Serialization: more intricate process-on-top-of-user-type support --- .../struct/languages/JavaCompiler.scala | 3 ++ .../components/EveryWriteIsExpression.scala | 43 +++++++++++++------ 2 files changed, 32 insertions(+), 14 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala index 696b70213..4cf31d3d6 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala @@ -568,6 +568,9 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def attrWriteStreamToStream(srcIo: String, dstIo: String) = out.puts(s"$dstIo.writeStream($srcIo);") + override def exprStreamToByteArray(io: String): String = + s"$io.toByteArray()" + def value2Const(s: String) = s.toUpperCase def idToStr(id: Identifier): String = { diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala index dd8674319..42a35a237 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala @@ -7,7 +7,7 @@ import io.kaitai.struct.format._ import scala.collection.mutable.ListBuffer -trait EveryWriteIsExpression extends LanguageCompiler with ObjectOrientedLanguage { +trait EveryWriteIsExpression extends LanguageCompiler with ObjectOrientedLanguage with EveryReadIsExpression { override def attrWrite(attr: AttrLikeSpec, id: Identifier, extraAttrs: ListBuffer[AttrSpec]): Unit = { attrParseIfHeader(id, attr.cond.ifExpr) @@ -164,20 +164,34 @@ trait EveryWriteIsExpression extends LanguageCompiler with ObjectOrientedLanguag case knownSizeType: UserTypeFromBytes => val rawId = RawIdentifier(id) val byteType = knownSizeType.bytes - byteType match { - case blt: BytesLimitType => - this match { - // case thisStore: AllocateAndStoreIO => - // val ourIO = thisStore.allocateIO(rawId, rep) - // Utils.addUniqueAttr(extraAttrs, AttrSpec(List(), ourIO, KaitaiStreamType)) - // privateMemberName(ourIO) - case thisLocal: AllocateIOLocalVar => - val ioFixed = thisLocal.allocateIOFixed(rawId, translator.translate(blt.size)) - attrUserTypeInstreamWrite(ioFixed, expr) - attrWriteStreamToStream(ioFixed, io) + byteType.process match { + case None => + byteType match { + case blt: BytesLimitType => + this match { + // case thisStore: AllocateAndStoreIO => + // val ourIO = thisStore.allocateIO(rawId, rep) + // Utils.addUniqueAttr(extraAttrs, AttrSpec(List(), ourIO, KaitaiStreamType)) + // privateMemberName(ourIO) + case thisLocal: AllocateIOLocalVar => + val ioFixed = thisLocal.allocateIOFixed(rawId, translator.translate(blt.size)) + attrUserTypeInstreamWrite(ioFixed, expr) + attrWriteStreamToStream(ioFixed, io) + } + case _ => + attrUserTypeInstreamWrite(io, expr) + } + case Some(process) => + byteType match { + case blt: BytesLimitType => + this match { + case thisLocal: AllocateIOLocalVar => + val ioFixed = thisLocal.allocateIOFixed(rawId, translator.translate(blt.size)) + attrUserTypeInstreamWrite(ioFixed, expr) + handleAssignment(rawId, exprStreamToByteArray(ioFixed), rep, isRaw) + attrBytesTypeWrite(rawId, byteType, io, extraAttrs, rep, isRaw) + } } - case _ => - attrUserTypeInstreamWrite(io, expr) } } } @@ -186,6 +200,7 @@ trait EveryWriteIsExpression extends LanguageCompiler with ObjectOrientedLanguag def attrBytesLimitWrite(io: String, expr: String, size: String, term: Int, padRight: Int): Unit def attrUserTypeInstreamWrite(io: String, expr: String): Unit def attrWriteStreamToStream(srcIo: String, dstIo: String): Unit + def exprStreamToByteArray(ioFixed: String): String def attrUnprocess(proc: ProcessExpr, varSrc: Identifier, varDest: Identifier): Unit From 0f3a2aab1ee0f52d11349d54edf645e6c9b8867e Mon Sep 17 00:00:00 2001 From: Mikhail Yakshin Date: Wed, 29 Mar 2017 10:44:33 +0300 Subject: [PATCH 17/90] JavaCompiler: implemented writing of repeat-eos --- .../scala/io/kaitai/struct/languages/JavaCompiler.scala | 5 +++++ .../languages/components/EveryWriteIsExpression.scala | 7 ++++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala index 4cf31d3d6..133b98dbd 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala @@ -336,6 +336,11 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.inc } + override def condRepeatEosHeader2(id: Identifier, io: String, dataType: DataType, needRaw: Boolean): Unit = { + out.puts(s"for (int _i = 0; _i < ${privateMemberName(id)}.size(); _i++) {") + out.inc + } + override def handleAssignmentRepeatEos(id: Identifier, expr: String): Unit = { out.puts(s"${privateMemberName(id)}.add($expr);") } diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala index 42a35a237..3f72694a7 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala @@ -15,7 +15,7 @@ trait EveryWriteIsExpression extends LanguageCompiler with ObjectOrientedLanguag attr.cond.repeat match { case RepeatEos => - condRepeatEosHeader(id, io, attr.dataType, needRaww(attr.dataType)) + condRepeatEosHeader2(id, io, attr.dataType, needRaww(attr.dataType)) attrWrite2(id, attr.dataType, io, extraAttrs, attr.cond.repeat, false) condRepeatEosFooter case RepeatExpr(repeatExpr: Ast.expr) => @@ -61,7 +61,7 @@ trait EveryWriteIsExpression extends LanguageCompiler with ObjectOrientedLanguag def writeExpr(id: Identifier, rep: RepeatSpec, isRaw: Boolean): String = { rep match { - case _: RepeatExpr => + case _: RepeatExpr | RepeatEos => translator.translate( Ast.expr.Subscript( Ast.expr.Name(Ast.identifier(idToStr(id))), @@ -212,5 +212,6 @@ trait EveryWriteIsExpression extends LanguageCompiler with ObjectOrientedLanguag } } - def condRepeatExprHeader2(id: Identifier, io: String, dataType: DataType, needRaw: Boolean, repeatExpr: Ast.expr): Unit = ??? + def condRepeatEosHeader2(id: Identifier, io: String, dataType: DataType, needRaw: Boolean): Unit + def condRepeatExprHeader2(id: Identifier, io: String, dataType: DataType, needRaw: Boolean, repeatExpr: Ast.expr): Unit } From e4a56f1e45020ae306b6068576b7ed2d1ddcab85 Mon Sep 17 00:00:00 2001 From: Mikhail Yakshin Date: Wed, 29 Mar 2017 19:46:17 +0300 Subject: [PATCH 18/90] Serialization: implemented switch types support, reworked JavaCompiler to use new KaitaiStruct.* abstract classes --- .../struct/languages/JavaCompiler.scala | 48 +++++--------- .../components/EveryWriteIsExpression.scala | 65 ++++++++++++++++--- .../struct/translators/JavaTranslator.scala | 13 ++-- 3 files changed, 81 insertions(+), 45 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala index 133b98dbd..0e4785df6 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala @@ -61,15 +61,9 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) "" } - val interfaces = if (config.readWrite) { - s"$kstructName.Readable, $kstructName.Writable" - } else { - s"$kstructName.Readable" - } - out.puts( s"public ${staticStr}class ${type2class(name)} " + - s"extends $kstructName implements $interfaces {" + s"extends $kstructNameFull {" ) out.inc @@ -179,17 +173,6 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } override def funcWriteHeader(curClass: ClassSpec): Unit = { - out.puts - out.puts(s"public void _write($kstreamName io) {") - out.inc - out.puts("if (this._io == null)") - out.inc - out.puts("this._io = io;") - out.dec - out.puts("_write();") - out.dec - out.puts("}") - out.puts out.puts("public void _write() {") out.inc @@ -601,17 +584,6 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def privateMemberName(id: Identifier): String = s"this.${idToStr(id)}" override def publicMemberName(id: Identifier) = idToStr(id) -} - -object JavaCompiler extends LanguageCompilerStatic - with UpperCamelCaseClasses - with StreamStructNames { - override def getTranslator(tp: TypeProvider, config: RuntimeConfig) = new JavaTranslator(tp) - - override def getCompiler( - tp: ClassTypeProvider, - config: RuntimeConfig - ): LanguageCompiler = new JavaCompiler(tp, config) def kaitaiType2JavaType(attrType: DataType): String = kaitaiType2JavaTypePrim(attrType) @@ -655,7 +627,7 @@ object JavaCompiler extends LanguageCompilerStatic case AnyType => "Object" case KaitaiStreamType => kstreamName - case KaitaiStructType => kstructName + case KaitaiStructType => kstructNameFull case t: UserType => types2class(t.name) case EnumType(name, _) => types2class(name) @@ -699,7 +671,7 @@ object JavaCompiler extends LanguageCompilerStatic case AnyType => "Object" case KaitaiStreamType => kstreamName - case KaitaiStructType => kstructName + case KaitaiStructType => kstructNameFull case t: UserType => type2class(t.name.last) case EnumType(name, _) => types2class(name) @@ -712,6 +684,20 @@ object JavaCompiler extends LanguageCompilerStatic def types2class(names: List[String]) = names.map(x => type2class(x)).mkString(".") + def kstructNameFull: String = + kstructName + "." + (if (config.readWrite) "ReadWrite" else "ReadOnly") +} + +object JavaCompiler extends LanguageCompilerStatic + with UpperCamelCaseClasses + with StreamStructNames { + override def getTranslator(tp: TypeProvider, config: RuntimeConfig) = new JavaTranslator(tp, config) + + override def getCompiler( + tp: ClassTypeProvider, + config: RuntimeConfig + ): LanguageCompiler = new JavaCompiler(tp, config) + override def kstreamName: String = "KaitaiStream" override def kstructName: String = "KaitaiStruct" } diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala index 3f72694a7..f8f15a75d 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala @@ -3,6 +3,7 @@ package io.kaitai.struct.languages.components import io.kaitai.struct.datatype.DataType import io.kaitai.struct.datatype.DataType._ import io.kaitai.struct.exprlang.Ast +import io.kaitai.struct.exprlang.Ast.expr import io.kaitai.struct.format._ import scala.collection.mutable.ListBuffer @@ -34,20 +35,20 @@ trait EveryWriteIsExpression extends LanguageCompiler with ObjectOrientedLanguag } def attrWrite2( - id: Identifier, - dataType: DataType, - io: String, - extraAttrs: ListBuffer[AttrSpec], - rep: RepeatSpec, - isRaw: Boolean - ): Unit = { + id: Identifier, + dataType: DataType, + io: String, + extraAttrs: ListBuffer[AttrSpec], + rep: RepeatSpec, + isRaw: Boolean + ): Unit = { dataType match { case t: UserType => attrUserTypeWrite(id, t, io, extraAttrs, rep, isRaw) case t: BytesType => attrBytesTypeWrite(id, t, io, extraAttrs, rep, isRaw) -// case SwitchType(on, cases) => -// attrSwitchTypeWrite(id, on, cases, io, extraAttrs, rep) + case SwitchType(on, cases) => + attrSwitchTypeWrite(id, on, cases, io, extraAttrs, rep) case t: StrFromBytesType => attrStrTypeWrite(id, t, io, extraAttrs, rep, isRaw) case t: EnumType => @@ -196,6 +197,52 @@ trait EveryWriteIsExpression extends LanguageCompiler with ObjectOrientedLanguag } } + def attrSwitchTypeWrite(id: Identifier, on: expr, cases: Map[expr, DataType], io: String, extraAttrs: ListBuffer[AttrSpec], rep: RepeatSpec) = { + switchStart(id, on) + + // Pass 1: only normal case clauses + var first = true + + cases.foreach { case (condition, dataType) => + condition match { + case SwitchType.ELSE_CONST => + // skip for now + case _ => + if (first) { + switchCaseFirstStart(condition) + first = false + } else { + switchCaseStart(condition) + } + attrWrite2(id, dataType, io, extraAttrs, rep, false) + switchCaseEnd() + } + } + + // Pass 2: else clause, if it is there + cases.foreach { case (condition, dataType) => + condition match { + case SwitchType.ELSE_CONST => + switchElseStart() + if (switchBytesOnlyAsRaw) { + dataType match { + case t: BytesType => + attrWrite2(RawIdentifier(id), dataType, io, extraAttrs, rep, false) + case _ => + attrWrite2(id, dataType, io, extraAttrs, rep, false) + } + } else { + attrWrite2(id, dataType, io, extraAttrs, rep, false) + } + switchElseEnd() + case _ => + // ignore normal case clauses + } + } + + switchEnd() + } + def attrPrimitiveWrite(io: String, expr: String, dt: DataType): Unit def attrBytesLimitWrite(io: String, expr: String, size: String, term: Int, padRight: Int): Unit def attrUserTypeInstreamWrite(io: String, expr: String): Unit diff --git a/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala index 358817db6..a86b46e75 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala @@ -1,14 +1,14 @@ package io.kaitai.struct.translators -import io.kaitai.struct.Utils -import io.kaitai.struct.exprlang.Ast -import io.kaitai.struct.exprlang.Ast._ +import io.kaitai.struct.{ClassTypeProvider, RuntimeConfig, Utils} import io.kaitai.struct.datatype.DataType import io.kaitai.struct.datatype.DataType._ +import io.kaitai.struct.exprlang.Ast +import io.kaitai.struct.exprlang.Ast._ import io.kaitai.struct.format.Identifier import io.kaitai.struct.languages.JavaCompiler -class JavaTranslator(provider: TypeProvider) extends BaseTranslator(provider) { +class JavaTranslator(provider: TypeProvider, config: RuntimeConfig) extends BaseTranslator(provider) { override def doIntLiteral(n: BigInt): String = { val literal = n.toString val suffix = if (n > Int.MaxValue) "L" else "" @@ -17,7 +17,10 @@ class JavaTranslator(provider: TypeProvider) extends BaseTranslator(provider) { } override def doArrayLiteral(t: DataType, value: Seq[expr]): String = { - val javaType = JavaCompiler.kaitaiType2JavaTypeBoxed(t) + // FIXME + val compiler = new JavaCompiler(provider.asInstanceOf[ClassTypeProvider], config) + + val javaType = compiler.kaitaiType2JavaTypeBoxed(t) val commaStr = value.map((v) => translate(v)).mkString(", ") s"new ArrayList<$javaType>(Arrays.asList($commaStr))" } From 44185e1e0f47d8b621c8a1fef4dc1f21b1ab2d8a Mon Sep 17 00:00:00 2001 From: Mikhail Yakshin Date: Thu, 30 Mar 2017 05:40:16 +0300 Subject: [PATCH 19/90] Added basic _check implementation, added checks for a proper number of members in a collection --- .../io/kaitai/struct/ClassCompiler.scala | 10 +++ .../struct/languages/JavaCompiler.scala | 19 +++++- .../languages/components/GenericChecks.scala | 68 +++++++++++++++++++ .../components/LanguageCompiler.scala | 4 ++ .../components/UniversalFooter.scala | 1 + 5 files changed, 100 insertions(+), 2 deletions(-) create mode 100644 shared/src/main/scala/io/kaitai/struct/languages/components/GenericChecks.scala diff --git a/shared/src/main/scala/io/kaitai/struct/ClassCompiler.scala b/shared/src/main/scala/io/kaitai/struct/ClassCompiler.scala index 3327457f8..013f70ab3 100644 --- a/shared/src/main/scala/io/kaitai/struct/ClassCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/ClassCompiler.scala @@ -65,6 +65,10 @@ class ClassCompiler( lang.funcWriteHeader(curClass) compileSeqWrite(curClass.seq, extraAttrs) lang.funcWriteFooter(curClass) + + lang.funcCheckHeader(curClass) + compileSeqCheck(curClass.seq) + lang.funcCheckFooter(curClass) } lang.classDestructorHeader(curClass.name, curClass.parentTypeName, topClassName) @@ -125,6 +129,12 @@ class ClassCompiler( } } + def compileSeqCheck(seq: List[AttrSpec]) = { + seq.foreach { (attr) => + lang.attrCheck(attr, attr.id) + } + } + def compileEnums(curClass: ClassSpec): Unit = curClass.enums.foreach { case(_, enumColl) => compileEnum(curClass, enumColl) } diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala index 0e4785df6..ad4551ff8 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala @@ -16,6 +16,7 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) with ObjectOrientedLanguage with EveryReadIsExpression with EveryWriteIsExpression + with GenericChecks with UniversalFooter with UniversalDoc with AllocateIOLocalVar @@ -41,8 +42,7 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts(s"package ${config.javaPackage};") } out.puts - out.puts(s"import io.kaitai.struct.$kstructName;") - out.puts(s"import io.kaitai.struct.$kstreamName;") + out.puts("import io.kaitai.struct.*;") out.puts out.puts("import java.io.IOException;") out.puts("import java.util.Arrays;") @@ -178,6 +178,12 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.inc } + override def funcCheckHeader(curClass: ClassSpec): Unit = { + out.puts + out.puts("public void _check() {") + out.inc + } + override def attrFixedContentsParse(attrName: Identifier, contents: String): Unit = { out.puts(s"${privateMemberName(attrName)} = $normalIO.ensureFixedContents($contents);") } @@ -559,6 +565,15 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def exprStreamToByteArray(io: String): String = s"$io.toByteArray()" + override def attrBasicCheck(checkExpr: String, actual: String, expected: String, msg: String): Unit = { + val msgStr = translator.translate(Ast.expr.Str(msg)) + + out.puts(s"if ($checkExpr)") + out.inc + out.puts(s"throw new ConsistencyError($msgStr, $actual, $expected);") + out.dec + } + def value2Const(s: String) = s.toUpperCase def idToStr(id: Identifier): String = { diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/GenericChecks.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/GenericChecks.scala new file mode 100644 index 000000000..545703c29 --- /dev/null +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/GenericChecks.scala @@ -0,0 +1,68 @@ +package io.kaitai.struct.languages.components +import io.kaitai.struct.datatype.DataType +import io.kaitai.struct.exprlang.Ast +import io.kaitai.struct.format._ + +trait GenericChecks extends LanguageCompiler with EveryReadIsExpression with EveryWriteIsExpression { + override def attrCheck(attr: AttrLikeSpec, id: Identifier): Unit = { + attrParseIfHeader(id, attr.cond.ifExpr) + + val io = normalIO + + attr.cond.repeat match { + case RepeatEos => + condRepeatEosHeader2(id, io, attr.dataType, needRaw(attr.dataType)) + attrCheck2(id, attr.dataType, io, attr.cond.repeat, false) + condRepeatEosFooter + case RepeatExpr(repeatExpr: Ast.expr) => + attrArraySizeCheck(id, repeatExpr) + condRepeatExprHeader2(id, io, attr.dataType, needRaw(attr.dataType), repeatExpr) + attrCheck2(id, attr.dataType, io, attr.cond.repeat, false) + condRepeatExprFooter + case RepeatUntil(untilExpr: Ast.expr) => + condRepeatUntilHeader(id, io, attr.dataType, needRaw(attr.dataType), untilExpr) + attrCheck2(id, attr.dataType, io, attr.cond.repeat, false) + condRepeatUntilFooter(id, io, attr.dataType, needRaww(attr.dataType), untilExpr) + case NoRepeat => + attrCheck2(id, attr.dataType, io, attr.cond.repeat, false) + } + + attrParseIfFooter(attr.cond.ifExpr) + } + + def attrCheck2(id: Identifier, dataType: DataType, io: String, repeat: RepeatSpec, isRaw: Boolean) = { + } + + def attrArraySizeCheck(id: Identifier, expectedSize: Ast.expr) = + attrBasicCheck( + Ast.expr.Attribute( + idToName(id), + Ast.identifier("size") + ), + expectedSize, + idToName(id).id.name + ) + + def attrBasicCheck(actual: Ast.expr, expected: Ast.expr, msg: String): Unit = + attrBasicCheck( + translator.translate(Ast.expr.Compare(actual, Ast.cmpop.NotEq, expected)), + translator.translate(actual), + translator.translate(expected), + msg + ) + + def attrBasicCheck(checkExpr: String, actual: String, expected: String, msg: String): Unit + + private + def idToName(id: Identifier): Ast.expr.Name = { + val str = id match { + case NumberedIdentifier(idx) => s"_${NumberedIdentifier.TEMPLATE}$idx" + case NamedIdentifier(name) => name + case RawIdentifier(innerId) => s"_raw_$innerId" + case IoStorageIdentifier(innerId) => s"_io_$innerId" + case InstanceIdentifier(name) => name + case SpecialIdentifier(name) => name + } + Ast.expr.Name(Ast.identifier(str)) + } +} diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala index 74780f1a3..fae3b7c62 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala @@ -80,6 +80,10 @@ abstract class LanguageCompiler( def funcWriteFooter(curClass: ClassSpec): Unit = ??? def attrWrite(attr: AttrLikeSpec, id: Identifier, extraAttrs: ListBuffer[AttrSpec]): Unit = ??? + def funcCheckHeader(curClass: ClassSpec): Unit = ??? + def funcCheckFooter(curClass: ClassSpec): Unit = ??? + def attrCheck(attr: AttrLikeSpec, id: Identifier): Unit = ??? + def attrFixedContentsParse(attrName: Identifier, contents: Array[Byte]): Unit def condIfSetNull(instName: Identifier): Unit = {} diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/UniversalFooter.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/UniversalFooter.scala index 9d2d13e78..cdde62343 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/UniversalFooter.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/UniversalFooter.scala @@ -16,6 +16,7 @@ trait UniversalFooter extends LanguageCompiler { def classFooter(name: String): Unit = universalFooter def classConstructorFooter: Unit = universalFooter override def funcWriteFooter(curClass: ClassSpec): Unit = universalFooter + override def funcCheckFooter(curClass: ClassSpec): Unit = universalFooter def condRepeatExprFooter = universalFooter def condRepeatEosFooter: Unit = universalFooter def condIfFooter(expr: expr): Unit = universalFooter From 769dabc336477a8997c1ec3ea2387d54a67f6c1a Mon Sep 17 00:00:00 2001 From: Mikhail Yakshin Date: Thu, 30 Mar 2017 11:05:17 +0300 Subject: [PATCH 20/90] Added String#to_b(encoding) and ByteArray#size methods + JavaTranslator implementations --- .../scala/io/kaitai/struct/translators/BaseTranslator.scala | 6 ++++++ .../scala/io/kaitai/struct/translators/JavaTranslator.scala | 2 ++ .../scala/io/kaitai/struct/translators/TypeDetector.scala | 6 ++++++ 3 files changed, 14 insertions(+) diff --git a/shared/src/main/scala/io/kaitai/struct/translators/BaseTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/BaseTranslator.scala index 4f4aa793c..7b4a56cc1 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/BaseTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/BaseTranslator.scala @@ -90,6 +90,10 @@ abstract class BaseTranslator(val provider: TypeProvider) extends TypeDetector(p case "last" => arrayLast(value) case "size" => arraySize(value) } + case _: BytesType => + attr.name match { + case "size" => bytesSize(value) + } case KaitaiStreamType => attr.name match { case "size" => kaitaiStreamSize(value) @@ -116,6 +120,7 @@ abstract class BaseTranslator(val provider: TypeProvider) extends TypeDetector(p case (_: StrType, "substring") => strSubstring(obj, args(0), args(1)) case (_: StrType, "to_i") => strToInt(obj, args(0)) case (_: BytesType, "to_s") => bytesToStr(translate(obj), args(0)) + case (_: StrType, "to_b") => strToBytes(translate(obj), args(0)) case _ => throw new TypeMismatchError(s"don't know how to call method '$methodName' of object type '$objType'") } } @@ -296,6 +301,7 @@ abstract class BaseTranslator(val provider: TypeProvider) extends TypeDetector(p doIfExp(value, Ast.expr.IntNum(1), Ast.expr.IntNum(0)) def intToStr(i: Ast.expr, base: Ast.expr): String def bytesToStr(bytesExpr: String, encoding: Ast.expr): String + def bytesSize(b: Ast.expr): String = ??? def strToBytes(strExpr: String, encoding: Ast.expr): String = ??? def strLength(s: Ast.expr): String def strReverse(s: Ast.expr): String diff --git a/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala index a86b46e75..9227f27a8 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala @@ -84,6 +84,8 @@ class JavaTranslator(provider: TypeProvider, config: RuntimeConfig) extends Base s"Long.toString(${translate(i)}, ${translate(base)})" override def bytesToStr(bytesExpr: String, encoding: Ast.expr): String = s"new String($bytesExpr, Charset.forName(${translate(encoding)}))" + override def bytesSize(b: Ast.expr): String = + s"${translate(b)}.length" override def strToBytes(strExpr: String, encoding: expr): String = s"($strExpr).getBytes(Charset.forName(${translate(encoding)}))" override def strLength(s: expr): String = diff --git a/shared/src/main/scala/io/kaitai/struct/translators/TypeDetector.scala b/shared/src/main/scala/io/kaitai/struct/translators/TypeDetector.scala index c3ec8caa5..3a5e0b84a 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/TypeDetector.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/TypeDetector.scala @@ -116,6 +116,11 @@ class TypeDetector(provider: TypeProvider) { case "size" => CalcIntType case _ => throw new TypeMismatchError(s"called invalid attribute '${attr.name}' on expression of type $valType") } + case _: BytesType => + attr.name match { + case "size" => CalcIntType + case _ => throw new TypeMismatchError(s"called invalid attribute '${attr.name}' on expression of type $valType") + } case KaitaiStreamType => attr.name match { case "size" => CalcIntType @@ -143,6 +148,7 @@ class TypeDetector(provider: TypeProvider) { (objType, methodName.name) match { case (_: StrType, "substring") => CalcStrType case (_: StrType, "to_i") => CalcIntType + case (_: StrType, "to_b") => CalcBytesType case _ => throw new RuntimeException(s"don't know how to call method '$methodName' of object type '$objType'") } } From 040b6890b33dbf6e3c48487aa0c375d99d170367 Mon Sep 17 00:00:00 2001 From: Mikhail Yakshin Date: Thu, 30 Mar 2017 11:05:45 +0300 Subject: [PATCH 21/90] Added limited byte / string sizing checks --- .../languages/components/GenericChecks.scala | 67 ++++++++++++++----- 1 file changed, 51 insertions(+), 16 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/GenericChecks.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/GenericChecks.scala index 545703c29..d8268f633 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/GenericChecks.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/GenericChecks.scala @@ -1,5 +1,6 @@ package io.kaitai.struct.languages.components import io.kaitai.struct.datatype.DataType +import io.kaitai.struct.datatype.DataType._ import io.kaitai.struct.exprlang.Ast import io.kaitai.struct.format._ @@ -31,18 +32,36 @@ trait GenericChecks extends LanguageCompiler with EveryReadIsExpression with Eve } def attrCheck2(id: Identifier, dataType: DataType, io: String, repeat: RepeatSpec, isRaw: Boolean) = { + dataType match { + case t: BytesType => + attrByteSizeCheck(id, t, exprByteArraySize(idToName(id))) + case st: StrFromBytesType => + attrByteSizeCheck(id, st.bytes, + exprByteArraySize(exprStrToBytes(idToName(id), st.encoding)) + ) + case _ => // no checks + } } - def attrArraySizeCheck(id: Identifier, expectedSize: Ast.expr) = + def attrArraySizeCheck(id: Identifier, expectedSize: Ast.expr): Unit = attrBasicCheck( - Ast.expr.Attribute( - idToName(id), - Ast.identifier("size") - ), + exprArraySize(idToName(id)), expectedSize, - idToName(id).id.name + idToMsg(id) ) + def attrByteSizeCheck(id: Identifier, t: BytesType, actualSize: Ast.expr): Unit = { + t match { + case blt: BytesLimitType => + attrBasicCheck( + actualSize, + blt.size, + idToMsg(id) + ) + case _ => // no checks + } + } + def attrBasicCheck(actual: Ast.expr, expected: Ast.expr, msg: String): Unit = attrBasicCheck( translator.translate(Ast.expr.Compare(actual, Ast.cmpop.NotEq, expected)), @@ -54,15 +73,31 @@ trait GenericChecks extends LanguageCompiler with EveryReadIsExpression with Eve def attrBasicCheck(checkExpr: String, actual: String, expected: String, msg: String): Unit private - def idToName(id: Identifier): Ast.expr.Name = { - val str = id match { - case NumberedIdentifier(idx) => s"_${NumberedIdentifier.TEMPLATE}$idx" - case NamedIdentifier(name) => name - case RawIdentifier(innerId) => s"_raw_$innerId" - case IoStorageIdentifier(innerId) => s"_io_$innerId" - case InstanceIdentifier(name) => name - case SpecialIdentifier(name) => name - } - Ast.expr.Name(Ast.identifier(str)) + def idToMsg(id: Identifier): String = id match { + case NumberedIdentifier(idx) => s"_${NumberedIdentifier.TEMPLATE}$idx" + case NamedIdentifier(name) => name + case RawIdentifier(innerId) => s"_raw_$innerId" + case IoStorageIdentifier(innerId) => s"_io_$innerId" + case InstanceIdentifier(name) => name + case SpecialIdentifier(name) => name } + + def idToName(id: Identifier): Ast.expr.Name = Ast.expr.Name(Ast.identifier(idToMsg(id))) + + def exprByteArraySize(name: Ast.expr) = + Ast.expr.Attribute( + name, + Ast.identifier("size") + ) + + def exprArraySize(name: Ast.expr) = exprByteArraySize(name) + + def exprStrToBytes(name: Ast.expr, encoding: String) = + Ast.expr.Call( + Ast.expr.Attribute( + name, + Ast.identifier("to_b") + ), + Seq(Ast.expr.Str(encoding)) + ) } From 28e8d062aad6f708e28c33bef0e914a6517e8c98 Mon Sep 17 00:00:00 2001 From: Mikhail Yakshin Date: Thu, 30 Mar 2017 23:51:23 +0300 Subject: [PATCH 22/90] Translators: added BytesType#first and #last, implemented for Java --- .../scala/io/kaitai/struct/translators/BaseTranslator.scala | 4 ++++ .../scala/io/kaitai/struct/translators/JavaTranslator.scala | 6 ++++++ .../scala/io/kaitai/struct/translators/TypeDetector.scala | 1 + 3 files changed, 11 insertions(+) diff --git a/shared/src/main/scala/io/kaitai/struct/translators/BaseTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/BaseTranslator.scala index 7b4a56cc1..430ecad30 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/BaseTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/BaseTranslator.scala @@ -92,6 +92,8 @@ abstract class BaseTranslator(val provider: TypeProvider) extends TypeDetector(p } case _: BytesType => attr.name match { + case "first" => bytesFirst(value) + case "last" => bytesLast(value) case "size" => bytesSize(value) } case KaitaiStreamType => @@ -301,6 +303,8 @@ abstract class BaseTranslator(val provider: TypeProvider) extends TypeDetector(p doIfExp(value, Ast.expr.IntNum(1), Ast.expr.IntNum(0)) def intToStr(i: Ast.expr, base: Ast.expr): String def bytesToStr(bytesExpr: String, encoding: Ast.expr): String + def bytesFirst(b: Ast.expr): String = ??? + def bytesLast(b: Ast.expr): String = ??? def bytesSize(b: Ast.expr): String = ??? def strToBytes(strExpr: String, encoding: Ast.expr): String = ??? def strLength(s: Ast.expr): String diff --git a/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala index 9227f27a8..cd6fbd9c1 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala @@ -84,6 +84,12 @@ class JavaTranslator(provider: TypeProvider, config: RuntimeConfig) extends Base s"Long.toString(${translate(i)}, ${translate(base)})" override def bytesToStr(bytesExpr: String, encoding: Ast.expr): String = s"new String($bytesExpr, Charset.forName(${translate(encoding)}))" + override def bytesFirst(b: Ast.expr): String = + s"${translate(b)}[0]" + override def bytesLast(b: Ast.expr): String = { + val v = translate(b) + s"$v[$v.length - 1]" + } override def bytesSize(b: Ast.expr): String = s"${translate(b)}.length" override def strToBytes(strExpr: String, encoding: expr): String = diff --git a/shared/src/main/scala/io/kaitai/struct/translators/TypeDetector.scala b/shared/src/main/scala/io/kaitai/struct/translators/TypeDetector.scala index 3a5e0b84a..785bd96ec 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/TypeDetector.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/TypeDetector.scala @@ -118,6 +118,7 @@ class TypeDetector(provider: TypeProvider) { } case _: BytesType => attr.name match { + case "first" | "last" => Int1Type(false) case "size" => CalcIntType case _ => throw new TypeMismatchError(s"called invalid attribute '${attr.name}' on expression of type $valType") } From 1b668977d7a2e5d2871f92efb8a36e9e2b134370 Mon Sep 17 00:00:00 2001 From: Mikhail Yakshin Date: Thu, 30 Mar 2017 23:52:13 +0300 Subject: [PATCH 23/90] GenericChecks: implemented tons of checks for BytesLimitType - boundary size checks, terminator inclusion checks, etc --- .../languages/components/GenericChecks.scala | 74 +++++++++++++++---- 1 file changed, 60 insertions(+), 14 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/GenericChecks.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/GenericChecks.scala index d8268f633..e6abe9bf7 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/GenericChecks.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/GenericChecks.scala @@ -44,7 +44,7 @@ trait GenericChecks extends LanguageCompiler with EveryReadIsExpression with Eve } def attrArraySizeCheck(id: Identifier, expectedSize: Ast.expr): Unit = - attrBasicCheck( + attrAssertEqual( exprArraySize(idToName(id)), expectedSize, idToMsg(id) @@ -53,23 +53,41 @@ trait GenericChecks extends LanguageCompiler with EveryReadIsExpression with Eve def attrByteSizeCheck(id: Identifier, t: BytesType, actualSize: Ast.expr): Unit = { t match { case blt: BytesLimitType => - attrBasicCheck( - actualSize, - blt.size, - idToMsg(id) - ) + if (blt.padRight.isDefined) { + // size must be "<= declared" + attrAssertLtE(actualSize, blt.size, idToMsg(id)) + blt.terminator.foreach { (term) => + if (blt.include) + attrAssertLastByte(id, term, idToMsg(id)) + } + } else { + blt.terminator match { + case Some(term) => + if (!blt.include) { + // size must be "<= (declared - 1)", i.e. "< declared" + attrAssertLt(actualSize, blt.size, idToMsg(id)) + } else { + // terminator is included into the string, so + // size must be "<= declared" + attrAssertLtE(actualSize, blt.size, idToMsg(id)) + attrAssertLastByte(id, term, idToMsg(id)) + } + case None => + // size must match declared size exactly + attrAssertEqual( + actualSize, + blt.size, + idToMsg(id) + ) + } + } + case btt: BytesTerminatedType => + if (btt.include) + attrAssertLastByte(id, btt.terminator, idToMsg(id)) case _ => // no checks } } - def attrBasicCheck(actual: Ast.expr, expected: Ast.expr, msg: String): Unit = - attrBasicCheck( - translator.translate(Ast.expr.Compare(actual, Ast.cmpop.NotEq, expected)), - translator.translate(actual), - translator.translate(expected), - msg - ) - def attrBasicCheck(checkExpr: String, actual: String, expected: String, msg: String): Unit private @@ -100,4 +118,32 @@ trait GenericChecks extends LanguageCompiler with EveryReadIsExpression with Eve ), Seq(Ast.expr.Str(encoding)) ) + + def attrAssertLastByte(actualId: Identifier, expectedLast: Int, msg: String): Unit = { + attrAssertEqual( + Ast.expr.Attribute( + idToName(actualId), + Ast.identifier("last") + ), + Ast.expr.IntNum(expectedLast), + msg + ) + } + + def attrAssertEqual(actual: Ast.expr, expected: Ast.expr, msg: String): Unit = + attrAssertCmp(actual, Ast.cmpop.NotEq, expected, msg) + + def attrAssertLtE(actual: Ast.expr, expected: Ast.expr, msg: String): Unit = + attrAssertCmp(actual, Ast.cmpop.Gt, expected, msg) + + def attrAssertLt(actual: Ast.expr, expected: Ast.expr, msg: String): Unit = + attrAssertCmp(actual, Ast.cmpop.GtE, expected, msg) + + def attrAssertCmp(actual: Ast.expr, op: Ast.cmpop, expected: Ast.expr, msg: String): Unit = + attrBasicCheck( + translator.translate(Ast.expr.Compare(actual, op, expected)), + translator.translate(actual), + translator.translate(expected), + msg + ) } From d73ec6b0d62cd24cb786c72249fd51b280364b35 Mon Sep 17 00:00:00 2001 From: Mikhail Yakshin Date: Wed, 5 Apr 2017 08:40:26 +0300 Subject: [PATCH 24/90] JavaCompiler: only use ReadOnly when _read is public, i.e. when debug is true --- .../scala/io/kaitai/struct/languages/JavaCompiler.scala | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala index b95309a65..4b2e573a5 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala @@ -700,8 +700,13 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) def types2class(names: List[String]) = names.map(x => type2class(x)).mkString(".") - def kstructNameFull: String = - kstructName + "." + (if (config.readWrite) "ReadWrite" else "ReadOnly") + def kstructNameFull: String = { + kstructName + ((config.debug, config.readWrite) match { + case (_, true) => ".ReadWrite" + case (true, false) => ".ReadOnly" + case (false, false) => "" + }) + } } object JavaCompiler extends LanguageCompilerStatic From 75729f7286efaf6bc45849d968a7a89f36a7a419 Mon Sep 17 00:00:00 2001 From: Petr Pucil Date: Sun, 27 Oct 2019 23:59:54 +0100 Subject: [PATCH 25/90] WIP --- .circleci/config.yml | 37 + .gitignore | 1 + .idea/.name | 2 +- .idea/compiler.xml | 22 - .idea/hydra.xml | 9 + .idea/misc.xml | 33 +- .idea/modules.xml | 12 +- .idea/modules/compiler-sources.iml | 28 +- .idea/modules/compilerJS-build.iml | 168 --- .idea/modules/compilerJS.iml | 34 - .idea/modules/compilerJVM-build.iml | 168 --- .idea/modules/compilerJVM.iml | 41 - .../kaitai-struct-compiler-common-build.iml | 182 +++ .../modules/kaitai-struct-compiler-common.iml | 27 + .idea/modules/kaitai-struct-compiler-js.iml | 35 + .idea/modules/kaitai-struct-compiler.iml | 39 + .idea/modules/root-build.iml | 168 --- .idea/modules/root.iml | 33 - .idea/sbt.xml | 7 +- .idea/scala_compiler.xml | 6 +- README.md | 196 ++- RELEASE_NOTES.md | 88 ++ build.sbt | 138 ++- js/README.md | 196 +++ js/package.json | 39 + .../main/scala/io/kaitai/struct/MainJs.scala | 23 +- .../struct/format/JavaScriptKSYParser.scala | 17 + .../scala/io/kaitai/struct/JavaMain.scala | 109 +- .../kaitai/struct/formats/JavaKSYParser.scala | 19 +- .../io/kaitai/struct/ErrorMessagesSpec.scala | 2 +- .../struct/datatype/SwitchType$Test.scala | 40 + .../io/kaitai/struct/exprlang/Ast$Test.scala | 50 + .../struct/exprlang/ExpressionsSpec.scala | 183 ++- .../struct/translators/TranslatorSpec.scala | 1092 ++++++++++------- .../translators/TypeDetector$Test.scala | 16 + lib_bintray.sh | 84 ++ project/build.properties | 2 +- project/plugins.sbt | 5 +- publish_deb_to_bintray.sh | 21 + publish_js_to_npm.sh | 10 + publish_zip_to_bintray.sh | 17 + .../io/kaitai/struct/ClassCompiler.scala | 346 +++++- .../io/kaitai/struct/ClassTypeProvider.scala | 85 +- .../scala/io/kaitai/struct/CompileLog.scala | 17 +- .../struct/ConstructClassCompiler.scala | 223 ++++ .../io/kaitai/struct/DocClassCompiler.scala | 84 ++ .../io/kaitai/struct/GoClassCompiler.scala | 95 ++ .../kaitai/struct/GraphvizClassCompiler.scala | 203 ++- .../io/kaitai/struct/HtmlClassCompiler.scala | 152 +++ .../scala/io/kaitai/struct/ImportList.scala | 13 + .../main/scala/io/kaitai/struct/JSON.scala | 13 +- .../src/main/scala/io/kaitai/struct/Log.scala | 6 + .../main/scala/io/kaitai/struct/Main.scala | 31 +- .../io/kaitai/struct/NimClassCompiler.scala | 299 +++++ .../io/kaitai/struct/RuntimeConfig.scala | 78 +- .../io/kaitai/struct/RustClassCompiler.scala | 104 ++ .../io/kaitai/struct/TypeProcessor.scala | 4 +- .../main/scala/io/kaitai/struct/Utils.scala | 36 + .../io/kaitai/struct/datatype/DataType.scala | 289 ++++- .../kaitai/struct/datatype/Endianness.scala | 71 +- .../io/kaitai/struct/datatype/KSError.scala | 44 + .../scala/io/kaitai/struct/exprlang/Ast.scala | 70 +- .../kaitai/struct/exprlang/Expressions.scala | 40 +- .../io/kaitai/struct/exprlang/Lexical.scala | 4 +- .../io/kaitai/struct/format/AttrSpec.scala | 132 +- .../io/kaitai/struct/format/ClassSpec.scala | 141 ++- .../io/kaitai/struct/format/ClassSpecs.scala | 25 + .../io/kaitai/struct/format/DocSpec.scala | 40 +- .../io/kaitai/struct/format/EnumSpec.scala | 18 +- .../kaitai/struct/format/EnumValueSpec.scala | 40 + .../io/kaitai/struct/format/Identifier.scala | 46 +- .../kaitai/struct/format/InstanceSpec.scala | 23 +- .../io/kaitai/struct/format/KSVersion.scala | 12 +- .../io/kaitai/struct/format/MemberSpec.scala | 31 + .../kaitai/struct/format/MetaDefaults.scala | 19 - .../io/kaitai/struct/format/MetaSpec.scala | 40 +- .../kaitai/struct/format/ParamDefSpec.scala | 41 + .../io/kaitai/struct/format/ParseUtils.scala | 53 +- .../io/kaitai/struct/format/ProcessExpr.scala | 57 +- .../io/kaitai/struct/format/RepeatSpec.scala | 51 + .../kaitai/struct/format/ValidationSpec.scala | 32 + .../struct/format/YAMLParseException.scala | 32 +- .../io/kaitai/struct/format/YAMLPath.scala | 6 + .../struct/languages/CSharpCompiler.scala | 304 ++++- .../kaitai/struct/languages/CppCompiler.scala | 895 ++++++++++---- .../kaitai/struct/languages/GoCompiler.scala | 602 +++++++++ .../struct/languages/JavaCompiler.scala | 501 ++++++-- .../struct/languages/JavaScriptCompiler.scala | 292 ++++- .../kaitai/struct/languages/LuaCompiler.scala | 412 +++++++ .../kaitai/struct/languages/PHPCompiler.scala | 178 ++- .../struct/languages/PerlCompiler.scala | 119 +- .../struct/languages/PythonCompiler.scala | 274 ++++- .../struct/languages/RubyCompiler.scala | 174 ++- .../struct/languages/RustCompiler.scala | 609 +++++++++ .../components/AllocateAndStoreIO.scala | 16 +- .../components/AllocateIOLocalVar.scala | 5 +- .../languages/components/CommonReads.scala | 89 ++ .../components/EveryReadIsExpression.scala | 238 ++-- .../components/EveryWriteIsExpression.scala | 92 +- .../languages/components/ExceptionNames.scala | 18 + .../languages/components/ExtraAttrs.scala | 66 + .../FixedContentsUsingArrayByteLiteral.scala | 10 +- .../struct/languages/components/GoReads.scala | 158 +++ .../languages/components/GoSwitchOps.scala | 59 + .../components/LanguageCompiler.scala | 66 +- .../components/LanguageCompilerStatic.scala | 9 +- .../components/NoNeedForFullClassPath.scala | 18 +- .../components/ObjectOrientedLanguage.scala | 18 + .../components/SingleOutputFile.scala | 22 +- .../languages/components/SwitchIfOps.scala | 109 ++ .../languages/components/SwitchOps.scala | 90 ++ .../components/UniversalFooter.scala | 5 +- .../languages/components/ValidateOps.scala | 34 + .../struct/precompile/CalculateSeqSizes.scala | 120 ++ .../kaitai/struct/precompile/Exceptions.scala | 15 +- .../struct/precompile/LoadImports.scala | 90 +- .../struct/precompile/ParentTypes.scala | 9 +- .../struct/precompile/ResolveTypes.scala | 30 +- .../precompile/SpecsValueTypeDerive.scala | 2 +- .../struct/precompile/TypeValidator.scala | 152 ++- .../struct/precompile/ValueTypesDeriver.scala | 8 +- .../translators/AbstractTranslator.scala | 20 + .../struct/translators/BaseTranslator.scala | 365 ++---- .../translators/ByteArraysAsTrueArrays.scala | 17 + .../struct/translators/CSharpTranslator.scala | 46 +- .../translators/CommonArraysAndCast.scala | 101 ++ .../struct/translators/CommonLiterals.scala | 80 ++ .../struct/translators/CommonMethods.scala | 140 +++ .../kaitai/struct/translators/CommonOps.scala | 67 + .../struct/translators/CommonSizeOf.scala | 40 + .../translators/ConstructTranslator.scala | 31 + .../struct/translators/CppTranslator.scala | 113 +- .../translators/ExpressionValidator.scala | 129 ++ .../struct/translators/GoTranslator.scala | 452 +++++++ .../translators/JavaScriptTranslator.scala | 40 +- .../struct/translators/JavaTranslator.scala | 77 +- .../struct/translators/LuaTranslator.scala | 159 +++ .../struct/translators/NimTranslator.scala | 102 ++ .../struct/translators/PHPTranslator.scala | 14 +- .../struct/translators/PerlTranslator.scala | 37 +- .../struct/translators/PythonTranslator.scala | 27 +- .../struct/translators/RubyTranslator.scala | 60 +- .../struct/translators/RustTranslator.scala | 131 ++ .../struct/translators/TypeDetector.scala | 317 +++-- .../struct/translators/TypeProvider.scala | 7 +- 145 files changed, 11909 insertions(+), 3116 deletions(-) create mode 100644 .circleci/config.yml delete mode 100644 .idea/compiler.xml create mode 100644 .idea/hydra.xml delete mode 100644 .idea/modules/compilerJS-build.iml delete mode 100644 .idea/modules/compilerJS.iml delete mode 100644 .idea/modules/compilerJVM-build.iml delete mode 100644 .idea/modules/compilerJVM.iml create mode 100644 .idea/modules/kaitai-struct-compiler-common-build.iml create mode 100644 .idea/modules/kaitai-struct-compiler-common.iml create mode 100644 .idea/modules/kaitai-struct-compiler-js.iml create mode 100644 .idea/modules/kaitai-struct-compiler.iml delete mode 100644 .idea/modules/root-build.iml delete mode 100644 .idea/modules/root.iml create mode 100644 js/README.md create mode 100644 js/package.json create mode 100644 jvm/src/test/scala/io/kaitai/struct/datatype/SwitchType$Test.scala create mode 100644 jvm/src/test/scala/io/kaitai/struct/exprlang/Ast$Test.scala create mode 100644 jvm/src/test/scala/io/kaitai/struct/translators/TypeDetector$Test.scala create mode 100644 lib_bintray.sh create mode 100755 publish_deb_to_bintray.sh create mode 100755 publish_js_to_npm.sh create mode 100755 publish_zip_to_bintray.sh create mode 100644 shared/src/main/scala/io/kaitai/struct/ConstructClassCompiler.scala create mode 100644 shared/src/main/scala/io/kaitai/struct/DocClassCompiler.scala create mode 100644 shared/src/main/scala/io/kaitai/struct/GoClassCompiler.scala create mode 100644 shared/src/main/scala/io/kaitai/struct/HtmlClassCompiler.scala create mode 100644 shared/src/main/scala/io/kaitai/struct/ImportList.scala create mode 100644 shared/src/main/scala/io/kaitai/struct/NimClassCompiler.scala create mode 100644 shared/src/main/scala/io/kaitai/struct/RustClassCompiler.scala create mode 100644 shared/src/main/scala/io/kaitai/struct/datatype/KSError.scala create mode 100644 shared/src/main/scala/io/kaitai/struct/format/EnumValueSpec.scala create mode 100644 shared/src/main/scala/io/kaitai/struct/format/MemberSpec.scala delete mode 100644 shared/src/main/scala/io/kaitai/struct/format/MetaDefaults.scala create mode 100644 shared/src/main/scala/io/kaitai/struct/format/ParamDefSpec.scala create mode 100644 shared/src/main/scala/io/kaitai/struct/format/RepeatSpec.scala create mode 100644 shared/src/main/scala/io/kaitai/struct/format/ValidationSpec.scala create mode 100644 shared/src/main/scala/io/kaitai/struct/languages/GoCompiler.scala create mode 100644 shared/src/main/scala/io/kaitai/struct/languages/LuaCompiler.scala create mode 100644 shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala create mode 100644 shared/src/main/scala/io/kaitai/struct/languages/components/CommonReads.scala create mode 100644 shared/src/main/scala/io/kaitai/struct/languages/components/ExceptionNames.scala create mode 100644 shared/src/main/scala/io/kaitai/struct/languages/components/ExtraAttrs.scala create mode 100644 shared/src/main/scala/io/kaitai/struct/languages/components/GoReads.scala create mode 100644 shared/src/main/scala/io/kaitai/struct/languages/components/GoSwitchOps.scala create mode 100644 shared/src/main/scala/io/kaitai/struct/languages/components/SwitchIfOps.scala create mode 100644 shared/src/main/scala/io/kaitai/struct/languages/components/SwitchOps.scala create mode 100644 shared/src/main/scala/io/kaitai/struct/languages/components/ValidateOps.scala create mode 100644 shared/src/main/scala/io/kaitai/struct/precompile/CalculateSeqSizes.scala create mode 100644 shared/src/main/scala/io/kaitai/struct/translators/AbstractTranslator.scala create mode 100644 shared/src/main/scala/io/kaitai/struct/translators/ByteArraysAsTrueArrays.scala create mode 100644 shared/src/main/scala/io/kaitai/struct/translators/CommonArraysAndCast.scala create mode 100644 shared/src/main/scala/io/kaitai/struct/translators/CommonLiterals.scala create mode 100644 shared/src/main/scala/io/kaitai/struct/translators/CommonMethods.scala create mode 100644 shared/src/main/scala/io/kaitai/struct/translators/CommonOps.scala create mode 100644 shared/src/main/scala/io/kaitai/struct/translators/CommonSizeOf.scala create mode 100644 shared/src/main/scala/io/kaitai/struct/translators/ConstructTranslator.scala create mode 100644 shared/src/main/scala/io/kaitai/struct/translators/ExpressionValidator.scala create mode 100644 shared/src/main/scala/io/kaitai/struct/translators/GoTranslator.scala create mode 100644 shared/src/main/scala/io/kaitai/struct/translators/LuaTranslator.scala create mode 100644 shared/src/main/scala/io/kaitai/struct/translators/NimTranslator.scala create mode 100644 shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 000000000..807c9a56b --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,37 @@ +version: 2 +defaults: &defaults + working_directory: ~/repo + docker: + - image: circleci/openjdk:8-jdk + environment: + JVM_OPTS: -Xmx3200m + TERM: dumb +jobs: + build: + <<: *defaults + steps: + - checkout + - restore_cache: + keys: + - v1-dependencies-{{ checksum "build.sbt" }} + # fallback to using the latest cache if no exact match is found + - v1-dependencies- + - run: cat /dev/null | sbt compile compilerJVM/stage fastOptJS buildNpmJsFile compilerJVM/universal:packageBin + - store_artifacts: + path: jvm/target/universal/kaitai-struct-compiler-*.zip + - save_cache: + paths: + - ~/.m2 + key: v1-dependencies--{{ checksum "build.sbt" }} + test: + <<: *defaults + steps: + - run: cat /dev/null | sbt test +workflows: + version: 2 + build_test_deploy: + jobs: + - build + - test: + requires: + - build diff --git a/.gitignore b/.gitignore index 9f85e31c2..e41b1404d 100644 --- a/.gitignore +++ b/.gitignore @@ -54,3 +54,4 @@ project/boot/ # Unpacking complete binaries for testing purposes /runnable +js/npm diff --git a/.idea/.name b/.idea/.name index fa80e1460..114a2e6ca 100644 --- a/.idea/.name +++ b/.idea/.name @@ -1 +1 @@ -KaitaiStruct \ No newline at end of file +kaitai-struct-compiler \ No newline at end of file diff --git a/.idea/compiler.xml b/.idea/compiler.xml deleted file mode 100644 index 96cc43efa..000000000 --- a/.idea/compiler.xml +++ /dev/null @@ -1,22 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/hydra.xml b/.idea/hydra.xml new file mode 100644 index 000000000..66eeb9a09 --- /dev/null +++ b/.idea/hydra.xml @@ -0,0 +1,9 @@ + + + + + \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml index 3d7f6d271..df60b6777 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -1,37 +1,6 @@ - - - - - - - - - - - - - - - - + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml index 0774832f9..d36870744 100644 --- a/.idea/modules.xml +++ b/.idea/modules.xml @@ -3,14 +3,10 @@ - - - - - - - - + + + + \ No newline at end of file diff --git a/.idea/modules/compiler-sources.iml b/.idea/modules/compiler-sources.iml index af9c5e700..d26043a40 100644 --- a/.idea/modules/compiler-sources.iml +++ b/.idea/modules/compiler-sources.iml @@ -4,24 +4,22 @@ - + - + - - - - - - - - - - - - - + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/modules/compilerJS-build.iml b/.idea/modules/compilerJS-build.iml deleted file mode 100644 index 95ce319e3..000000000 --- a/.idea/modules/compilerJS-build.iml +++ /dev/null @@ -1,168 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/modules/compilerJS.iml b/.idea/modules/compilerJS.iml deleted file mode 100644 index ba43a620c..000000000 --- a/.idea/modules/compilerJS.iml +++ /dev/null @@ -1,34 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/modules/compilerJVM-build.iml b/.idea/modules/compilerJVM-build.iml deleted file mode 100644 index 169767c13..000000000 --- a/.idea/modules/compilerJVM-build.iml +++ /dev/null @@ -1,168 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/modules/compilerJVM.iml b/.idea/modules/compilerJVM.iml deleted file mode 100644 index fc02511df..000000000 --- a/.idea/modules/compilerJVM.iml +++ /dev/null @@ -1,41 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/modules/kaitai-struct-compiler-common-build.iml b/.idea/modules/kaitai-struct-compiler-common-build.iml new file mode 100644 index 000000000..1e4d140fc --- /dev/null +++ b/.idea/modules/kaitai-struct-compiler-common-build.iml @@ -0,0 +1,182 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/modules/kaitai-struct-compiler-common.iml b/.idea/modules/kaitai-struct-compiler-common.iml new file mode 100644 index 000000000..43c886e31 --- /dev/null +++ b/.idea/modules/kaitai-struct-compiler-common.iml @@ -0,0 +1,27 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/modules/kaitai-struct-compiler-js.iml b/.idea/modules/kaitai-struct-compiler-js.iml new file mode 100644 index 000000000..5f52a9ff6 --- /dev/null +++ b/.idea/modules/kaitai-struct-compiler-js.iml @@ -0,0 +1,35 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/modules/kaitai-struct-compiler.iml b/.idea/modules/kaitai-struct-compiler.iml new file mode 100644 index 000000000..a4330440c --- /dev/null +++ b/.idea/modules/kaitai-struct-compiler.iml @@ -0,0 +1,39 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/modules/root-build.iml b/.idea/modules/root-build.iml deleted file mode 100644 index ad68dd6c8..000000000 --- a/.idea/modules/root-build.iml +++ /dev/null @@ -1,168 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/modules/root.iml b/.idea/modules/root.iml deleted file mode 100644 index f8234bc36..000000000 --- a/.idea/modules/root.iml +++ /dev/null @@ -1,33 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/sbt.xml b/.idea/sbt.xml index e6a1c1e56..1d5e6f308 100644 --- a/.idea/sbt.xml +++ b/.idea/sbt.xml @@ -3,21 +3,18 @@ diff --git a/.idea/scala_compiler.xml b/.idea/scala_compiler.xml index f6bb54cd5..3bf45bc8b 100644 --- a/.idea/scala_compiler.xml +++ b/.idea/scala_compiler.xml @@ -2,10 +2,10 @@ diff --git a/README.md b/README.md index 757e31312..29959e70a 100644 --- a/README.md +++ b/README.md @@ -1,125 +1,34 @@ # Kaitai Struct: compiler -This project is an official reference compiler for [Kaitai Struct](https://github.com/kaitai-io/kaitai_struct) project. +This project is an official reference compiler for [Kaitai Struct](http://kaitai.io) project. -Kaitai Struct is a declarative language used for describe various +Kaitai Struct is a declarative language used to describe various binary data structures, laid out in files or in memory: i.e. binary file formats, network stream packet formats, etc. The main idea is that a particular format is described in Kaitai Struct language (`.ksy` files) only once and then can be compiled with this compiler into source files in one of the supported programming -languages. These modules will include a generated code for a parser +languages. These modules will include the generated code for a parser that can read described data structure from a file / stream and give access to it in a nice, easy-to-comprehend API. -Please refer to [documentation in Kaitai Struct project](https://github.com/kaitai-io/kaitai_struct) -for details on `.ksy` files and general usage patterns. +## Further information -## Trying without install +If you're looking for information on: -Kaitai Struct compiler can be tried instantly, without any downloads -and installation, at - -http://kaitai.io/repl - -Note that this implementation uses the same reference code as in this -repository and executes totally on a client side, without any queries -to server backend. - -## Downloading and installing - -### Linux .deb builds (Debian/Ubuntu) - -There is an official .deb repository available. The repository is hosted -at BinTray and signed with BinTray GPG key (`379CE192D401AB61`), so it's -necessary to import that key first if your box haven't used any BinTray -repositories beforehand: - -```shell -echo "deb https://dl.bintray.com/kaitai-io/debian jessie main" | sudo tee /etc/apt/sources.list.d/kaitai.list -sudo apt-key adv --keyserver hkp://pool.sks-keyservers.net --recv 379CE192D401AB61 -sudo apt-get update -sudo apt-get install kaitai-struct-compiler -``` - -### Windows builds - -An official `.msi` installer build is available to download at - -https://bintray.com/kaitai-io/universal/kaitai-struct-compiler/_latestVersion - -### Universal builds - -Basically, everything that can run Java can use so called "universal" -builds: a .zip file that includes all the required .jar files bundled -and launcher scripts for UNIX/Windows systems. No installation -required, one can just unpack and run it. Available at download also at - -https://bintray.com/kaitai-io/universal/kaitai-struct-compiler/_latestVersion - -### Source code - -If you're interested in developing compiler itself, you can check out -source code in repository: - - git clone https://github.com/kaitai-io/kaitai_struct_compiler - -See the [developer documentation](http://doc.kaitai.io/developers.html) for -general pointers on how to proceed with the source code then. - -## Usage - -`kaitai-struct-compiler [options] ...` - -Alternatively, a symlink `ksc` is provided and can be used everywhere -just as full name. - -Common options: - -* `...` — source files (.ksy) -* `-t | --target ` — target languages (`cpp_stl`, - `csharp`, `java`, `javascript`, `perl`, `php`, `python`, `ruby`, `all`) - * `all` is a special case: it compiles all possible target - languages, creating language-specific directories (as per language - identifiers) inside output directory, and then creating output - module(s) for each language starting from there -* `-d | --outdir ` — output directory - (filenames will be auto-generated) - -Language-specific options: - -* `--dot-net-namespace ` — .NET namespace (C# only, default: Kaitai) -* `--java-package ` — Java package (Java only, default: root package) -* `--php-namespace ` — PHP namespace (PHP only, default: root package) - -Misc options: - -* `--verbose` — verbose output -* `--help` — display usage information and exit -* `--version` — output version information and exit - -A few examples, given that file `foo.ksy` exists in current directory -and describes format with ID `foo`: - -* `kaitai-struct-compiler -t python foo.ksy` — compile format in - `foo.ksy`, write output in current directory to file `foo.py` -* `kaitai-struct-compiler -t java foo.ksy` — compile format in - `foo.ksy`, create "src" subdir in current one and write output in - `src/Foo.java` -* `kaitai-struct-compiler -t java --java-package org.example foo.ksy` - — compile format in `foo.ksy`, create "src/org/example" subdir tree - in current one and write output in `src/org/example/Foo.java`; - resulting file will bear correct Java package clause. -* `kaitai-struct-compiler -t all -d /tmp/out --java-package org.example foo.ksy` - — compile format in `foo.ksy`, creating a hierarchy of files: - * `/tmp/out/java/src/org/example/Foo.java` - * `/tmp/out/python/foo.py` - * `/tmp/out/ruby/foo.rb` +* Kaitai Struct language itself (`.ksy` files, general usage patterns) + — refer to the [user guide](http://doc.kaitai.io/user_guide.html). +* How to download and install Kaitai Struct — see the + [downloads](http://kaitai.io/#download). +* How to build the compiler, run the test suite, and join the + development — see the [developer memo](http://doc.kaitai.io/developers.html). ## Licensing -Kaitai Struct compiler itself is copyright (C) 2015-2017 Kaitai +### Main code + +Kaitai Struct compiler itself is copyright (C) 2015-2019 Kaitai Project. This program is free software: you can redistribute it and/or modify @@ -135,7 +44,76 @@ General Public License for more details. You should have received a copy of the GNU General Public License along with this program. If not, see . -Note that it applies only to compiler itself, not `.ksy` input files -that one supplies in normal process of compilation, nor to compiler's -output files — that consitutes normal usage process and you obviously -keep copyright to both. +### FastParse + +Portions of Kaitai Struct compiler are loosely based on +[pythonparse](https://github.com/lihaoyi/fastparse/tree/master/pythonparse/shared/src/main/scala/pythonparse) +from [FastParse](http://www.lihaoyi.com/fastparse/) and are copyright +(c) 2014 Li Haoyi (haoyi.sg@gmail.com). + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +### XMLUtils code + +Portions of Kaitai Struct compiler are based on `scala/xml/Utility.scala` from [Scala XML](https://github.com/scala/scala-xml). + +Copyright (c) 2002-2017 EPFL +Copyright (c) 2011-2017 Lightbend, Inc. + +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. +* Neither the name of the EPFL nor the names of its contributors may be + used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +THE POSSIBILITY OF SUCH DAMAGE. + +### Libraries used + +Kaitai Struct compiler depends on the following libraries: + +* [scopt](https://github.com/scopt/scopt) — MIT license +* [fastparse](http://www.lihaoyi.com/fastparse/) — MIT license +* [snakeyaml](https://bitbucket.org/asomov/snakeyaml) — Apache 2.0 license + +--- + +Note that these clauses only apply only to compiler itself, not `.ksy` +input files that one supplies in normal process of compilation, nor to +compiler's output files — that consitutes normal usage process and you +obviously keep copyright to both. diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index b96e26291..5b044839c 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,3 +1,91 @@ +# 0.9 (TBD) + +* New targets support: + * Python with [Construct library](https://construct.readthedocs.io) +* Expression language: + * New methods: + * byte arrays: `length` + +# 0.8 (2018-02-05) + +* New target languages: + * Lua (96% tests pass score) + * initial support for Go (15% tests pass score) +* New ksy features: + * Switchable default endianness: `meta/endian` can now contain a + switch-like structure (with `switch-on` and `cases`), akin to + switchable types + ([docs](http://doc.kaitai.io/user_guide.html#calc-endian)). + * Parametric user-defined types: one can use `type: my_type(arg1, + arg2, arg3)` to pass arguments into user type + ([docs](http://doc.kaitai.io/user_guide.html#param-types)). + * Custom processing types: one can use `process: + my_process_name(arg1, arg2, arg3)` to invoke custom processing + routine, implemented in imperative language + ([docs](http://doc.kaitai.io/user_guide.html#custom-process)). + * In repetitions, index of current repetition can be accessed using + `_index` in expressions + ([docs](http://doc.kaitai.io/user_guide.html#repeat-index)). + * Verbose enums: now one can specify documentation and other useful + information relevant to enums using verbose enum declaration + format + ([docs](http://doc.kaitai.io/user_guide.html#verbose-enums)). + * `meta/xref` key can be used for adding cross-references of a + format specifications (like relevant RFC entries, Wikidata + entries, ISO / IEEE / JIS / DIN / GOST standard numbers, PRONOM + identifiers, etc). +* General compilation improvements: + * Imports/includes for all languages are now managed properly, no + duplicate / unnecessary imports should be added + * Python: basic docstring support + * More strict ksy precompile checks (less likely to accept ksy that + will result in non-compilable code), better error messages +* CLI options: + * Python target now allows to specify package with `--python-package` + * Java target now allows custom KaitaiStream implementations and + thus allows to specify default implementation for `fromFile(...)` + using `--java-from-file-class`. +* Expression language: + * New methods: + * floats: `to_i` + * arrays: `min`, `max` + * Added byte array comparison +* Packaging / infrastructure improvements: + * ksc is now available as + [npm package](https://www.npmjs.com/package/kaitai-struct-compiler/), + which now a build dependency of a + [web IDE](https://ide.kaitai.io/) +* Runtime API changes: + * C++: now requires `KS_STR_ENCODING_ICONV` or + `KS_STR_ENCODING_NONE` to be defined to how to handle string + encodings + * Java: `KaitaiStream` is now an interface, and there are two + distinct classes which implement it: + * `ByteBufferKaitaiStream` provides KaitaiStream backed + `ByteBuffer` (and thus using memory-mapped files) + * `RandomAccessFileKaitaiStream` provides KaitaiStream backed by + `RandomAccessFile` (and thus uses normal OS read calls, as it + was done in older KaitaiStruct circa v0.5) + * JavaScript: Error classes are now subclasses of `KaitaiStream` and + were renamed in the following way: `KaitaiUnexpectedDataError` -> + `KaitaiStream`.`UnexpectedDataError` +* Major bugfixes: + * C++: adjusted to made compatible with OS X and Windows MSVC builds + * Fixed broken generation of byte array literals with high 8-bit set + in some targets + * Fixed float literals parsing, fixed larger integer keys YAML parsing + * Fixed inconsistency of debug mode vs non-debug mode behavior for + `repeat-*` + * Fixed chain of relative imports bug: now all relative imports work + always relative to the file being processed, not to current + compiler's dir + * Many problems with switching: invalid common type inferring, + invalid code being generated, added failsafe `if`-based + implementations for languages which do not support switching over + all possible types. + * Fixed most memory leaks in C++ (only exception-related leaks are + left now) + # 0.7 (2017-03-22) * New ksy features: diff --git a/build.sbt b/build.sbt index 67f0708aa..81b22f107 100644 --- a/build.sbt +++ b/build.sbt @@ -1,12 +1,16 @@ import java.io.File +import java.nio.charset.Charset +import java.nio.file.Files import com.typesafe.sbt.packager.linux.{LinuxPackageMapping, LinuxSymlink} import sbt.Keys._ resolvers += Resolver.sonatypeRepo("public") -val VERSION = "0.7" -val TARGET_LANGS = "C++/STL, C#, Java, JavaScript, Perl, PHP, Python, Ruby" +val NAME = "kaitai-struct-compiler" +val VERSION = "0.9-SNAPSHOT" +val TARGET_LANGS = "C++/STL, C#, Java, JavaScript, Lua, Perl, PHP, Python, Ruby" +val UTF8 = Charset.forName("UTF-8") lazy val root = project.in(file(".")). aggregate(compilerJS, compilerJVM). @@ -16,26 +20,21 @@ lazy val root = project.in(file(".")). ) lazy val compiler = crossProject.in(file(".")). - enablePlugins(BuildInfoPlugin). enablePlugins(JavaAppPackaging). settings( organization := "io.kaitai", - name := "kaitai-struct-compiler", - version := VERSION, + version := sys.env.getOrElse("KAITAI_STRUCT_VERSION", VERSION), licenses := Seq(("GPL-3.0", url("https://opensource.org/licenses/GPL-3.0"))), - scalaVersion := "2.11.7", - buildInfoKeys := Seq[BuildInfoKey](name, version, scalaVersion, sbtVersion), - buildInfoPackage := "io.kaitai.struct", - buildInfoOptions += BuildInfoOption.BuildTime, + scalaVersion := "2.12.4", // Repo publish options - publishTo <<= version { (v: String) => + publishTo := version { (v: String) => val nexus = "https://oss.sonatype.org/" if (v.trim.endsWith("SNAPSHOT")) Some("snapshots" at nexus + "content/repositories/snapshots") else Some("releases" at nexus + "service/local/staging/deploy/maven2") - }, + }.value, pomExtra := http://kaitai.io @@ -53,16 +52,21 @@ lazy val compiler = crossProject.in(file(".")). , + generateVersion := generateVersionTask.value, // register manual sbt command + sourceGenerators in Compile += generateVersionTask.taskValue, // update automatically on every rebuild + libraryDependencies ++= Seq( - "com.lihaoyi" %%% "fastparse" % "0.4.1", + "com.github.scopt" %%% "scopt" % "3.6.0", + "com.lihaoyi" %%% "fastparse" % "1.0.0", "org.yaml" % "snakeyaml" % "1.16" ) ). jvmSettings( + name := NAME, + mainClass in Compile := Some("io.kaitai.struct.JavaMain"), libraryDependencies ++= Seq( - "org.scalatest" %% "scalatest" % "2.2.6" % "test", - "com.github.scopt" %% "scopt" % "3.4.0" + "org.scalatest" %% "scalatest" % "3.0.1" % "test" ), testOptions in Test += Tests.Argument(TestFrameworks.ScalaTest, "-u", "target/test_out"), @@ -111,18 +115,25 @@ lazy val compiler = crossProject.in(file(".")). } }, - // Hack: we need /usr/share/kaitai-struct (the format directory) to be - // created as empty dir and packaged in compiler package, to be filled in - // with actual repository contents by "kaitai-struct-formats" package. - // "jvm/src/main/resources" is guaranteed to be an empty directory. - linuxPackageMappings += LinuxPackageMapping(Map( - new File("jvm/src/main/resources") -> "/usr/share/kaitai-struct" - )), + // We need /usr/share/kaitai-struct (the format directory) to be created as + // empty dir and packaged in compiler package, to be filled in with actual + // repository contents by "kaitai-struct-formats" package. + linuxPackageMappings += packageTemplateMapping("/usr/share/kaitai-struct")(), // Remove all "maintainer scripts", such as prerm/postrm/preinst/postinst: default // implementations create per-package virtual user that we won't use anyway maintainerScripts in Debian := Map(), + // Work around new Debian defaults and sbt-native-packager defaults, which + // build .deb packages that appear to be incompatible with older Debian/Ubuntu's + // dpkg and are not accepted by BinTray. + // + // For more information, see + // https://github.com/sbt/sbt-native-packager/issues/1067 + debianNativeBuildOptions in Debian := Seq("-Zgzip", "-z3"), + + debianPackageDependencies := Seq("java8-runtime-headless"), + packageSummary in Linux := s"compiler to generate binary data parsers in $TARGET_LANGS", packageSummary in Windows := "Kaitai Struct compiler", packageDescription in Linux := @@ -147,8 +158,91 @@ lazy val compiler = crossProject.in(file(".")). maintainer in Debian := "Mikhail Yakshin " ). jsSettings( - // Add JS-specific settings here + name := NAME + "-js", + buildNpmJsFile := buildNpmJsFileTask.value, + buildNpmPackage := buildNpmPackageTask.value ) lazy val compilerJVM = compiler.jvm lazy val compilerJS = compiler.js + +lazy val generateVersion = taskKey[Seq[File]]("generateVersion") +lazy val generateVersionTask = Def.task { + // Generate contents of Version.scala + val contents = s"""package io.kaitai.struct + | + |object Version { + | val name = "${name.value}" + | val version = "${version.value}" + | val gitCommit = "${sys.env.getOrElse("GIT_COMMIT", "GIT_COMMIT not defined")}" + | val gitTime = "${sys.env.getOrElse("GIT_DATE_ISO", "GIT_DATE_ISO not defined")}" + |} + |""".stripMargin + + // Update Version.scala file, if needed + val file = (sourceManaged in Compile).value / "version" / "Version.scala" + println(s"Version file generated: $file") + IO.write(file, contents) + Seq(file) +} + +/** + * Builds JavaScript output file to be packaged as part of NPM package. + * Essentially wraps raw JavaScript output into AMD-style exports. + */ +lazy val buildNpmJsFile = taskKey[Seq[File]]("buildNpmJsFile") +lazy val buildNpmJsFileTask = Def.task { + val compiledFile = target.value / "scala-2.12" / s"${name.value}-fastopt.js" + println(s"buildNpmJsFile: reading $compiledFile") + val compiledFileContents = IO.read(compiledFile, UTF8) + + val fileWithExports = + s"""(function (root, factory) { + | if (typeof define === 'function' && define.amd) { + | define([], factory); + | } else if (typeof module === 'object' && module.exports) { + | module.exports = factory(); + | } else { + | root.KaitaiStructCompiler = factory(); + | } + |}(this, function () { + | + |var exports = {}; + |var __ScalaJSEnv = { exportsNamespace: exports }; + | + |$compiledFileContents + | + |return exports.io.kaitai.struct.MainJs; + | + |})); + """.stripMargin + + val targetFile = new File(s"js/npm/${name.value}.js") + println(s"buildNpmJsFile: writing $targetFile with AMD exports") + IO.write(targetFile, fileWithExports, UTF8) + Seq(targetFile) +} + +lazy val buildNpmPackage = taskKey[Seq[File]]("buildNpmPackage") +lazy val buildNpmPackageTask = Def.task { + val licenseFile = new File("js/npm/LICENSE") + val readMeFile = new File("js/npm/README.md") + val packageJsonFile = new File("js/npm/package.json") + + Files.copy(new File("LICENSE").toPath, licenseFile.toPath) + Files.copy(new File("js/README.md").toPath, readMeFile.toPath) + + val packageJsonTmpl = IO.read(new File("js/package.json"), UTF8) + val packageJsonContents = packageJsonTmpl.replaceFirst( + "\"version\": \".*?\"", + "\"version\": \"" + version.value + "\"" + ) + + IO.write(packageJsonFile, packageJsonContents, UTF8) + + Seq( + licenseFile, + readMeFile, + packageJsonFile + ) +} diff --git a/js/README.md b/js/README.md new file mode 100644 index 000000000..5ad3c52ad --- /dev/null +++ b/js/README.md @@ -0,0 +1,196 @@ +## Kaitai Struct compiler in JavaScript + +This project is a official reference Kaitai Struct compiler, compiled +for JavaScript environments. + +Kaitai Struct is a declarative language used for describe various +binary data structures, laid out in files or in memory: i.e. binary +file formats, network stream packet formats, etc. + +The main idea is that a particular format is described in Kaitai +Struct language only once and then can be compiled with into +source files in one of the supported programming languages. These +modules will include a generated code for a parser that can read +described data structure from a file / stream and give access to it in +a nice, easy-to-comprehend API. + +For more info on Kaitai Struct, please refer to http://kaitai.io/ + +Note that reference Kaitai Struct compiler is written Scala, and thus +can be compiled for a variety of platforms, such as JVM, JavaScript +and native binaries. This package is compiled to be run JavaScript +environments. + +Currently, this JavaScript build offers only programmatic API, so it +is generally suited for developers of tools that use Kaitai Struct +(i.e. interactive compilers, visualizers, loaders, IDEs that integrate +the compiler). If you: + +* just look for a way to try out Kaitai Struct => try + [Kaitai Struct Web IDE](https://ide.kaitai.io/), which uses this + compiler internally +* want to load .ksy file transparently in your JavaScript code + (compiling it on the fly) => use + [Kaitai Struct loader for JavaScript](https://github.com/kaitai-io/kaitai-struct-loader). +* want to integrate compiler into your build flow => use a JVM-based + desktop compiler, which can be called as a command-line utility + +## Installation + +We publish two versions of the compiler to npm: + - A stable one, this includes the latest stable, released compiler. This is the default ("latest") version. + ``` + npm install kaitai-struct-compiler + ``` + - The other is the latest snapshot version which follows our master branch. This version is tagged as `@next`. + ``` + npm install kaitai-struct-compiler@next + ``` + +### Example project + +Our [examples repository](https://github.com/kaitai-io/kaitai_struct_examples) contains a few examples how to use the compiler. + +## Plugging in + +We publish the compiler as an [UMD module](https://github.com/umdjs/umd), so it works from various environments, including server-side (eg. node) and client-side (eg. web browser) ones. + +Note: currently we don't publish the compiler as standard ES module. This will probably change in the future. If you need ES module please comment on [this issue](https://github.com/kaitai-io/kaitai_struct/issues/180). + +### node + +```javascript +var KaitaiStructCompiler = require("kaitai-struct-compiler"); +var compiler = new KaitaiStructCompiler(); +``` + +### browser using script tags + +```html + + +``` + +### browser using AMD loader (eg. require.js) + +```html + + +``` + +## Usage + +### Basic usage of compile method + +```javascript +var ksyYaml = fs.readFileSync("zip.ksy"); /* string */ +var ksy = YAML.parse(ksyYaml); /* JS object */ +compiler.compile("javascript", ksy, null, false /* debugMode */).then(function(files) { + console.log("Compiled filenames: " + Object.keys(files).join(", ")); + console.log("Content of Zip.js file: " + files["Zip.js"]); +}); +``` + +### Getting compiler information + +```javascript +console.log("Version: " + compiler.version); +console.log("Build date: " + compiler.buildDate); +console.log("Supported languages: " + compiler.languages.join(", ")); +``` + +### Handle imports + +```javascript +var yamlImporter = { + importYaml: function(name, mode) { + console.log(" -> Import yaml called with name '" + name + "' and mode '" + mode + "'."); + var importKsyYaml = fs.readFileSync(name + ".ksy"); /* string */ + var importKsy = YAML.parse(importKsyYaml); /* JS object */ + return Promise.resolve(importKsy); + } +}; + +yamlImporter.importYaml("import_outer.ksy").then(function(ksy) { + compiler.compile("javascript", ksy, yamlImporter, false /* debugMode */).then(function(files) { + console.log("Compiled filenames: " + Object.keys(files).join(", ")); + }); +}); +``` + +### Debug mode + +You can compile in debug mode which adds `_debug` property to every object and this `_debug` object contains the start and end offsets of the parsed fields so you can find bytes of the field in the original binary. + +## Copyrights and licensing + +### Main code + +Kaitai Struct compiler itself is copyright (C) 2015-2017 Kaitai +Project. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or (at +your option) any later version. + +This program is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . + +### FastParse + +Portions of Kaitai Struct compiler are loosely based on +[pythonparse](https://github.com/lihaoyi/fastparse/tree/master/pythonparse/shared/src/main/scala/pythonparse) +from [FastParse](http://www.lihaoyi.com/fastparse/) and are copyright +(c) 2014 Li Haoyi (haoyi.sg@gmail.com). + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +### Libraries used + +Kaitai Struct compiler depends on the following libraries: + +* [scopt](https://github.com/scopt/scopt) — MIT license +* [fastparse](http://www.lihaoyi.com/fastparse/) — MIT license +* [snakeyaml](https://bitbucket.org/asomov/snakeyaml) — Apache 2.0 license + +--- + +Note that these clauses only apply only to compiler itself, not `.ksy` +input files that one supplies in normal process of compilation, nor to +compiler's output files — that consitutes normal usage process and you +obviously keep copyright to both. diff --git a/js/package.json b/js/package.json new file mode 100644 index 000000000..c2c2ae68d --- /dev/null +++ b/js/package.json @@ -0,0 +1,39 @@ +{ + "author": { + "name": "Kaitai team", + "url": "https://github.com/orgs/kaitai-io/people" + }, + "bugs": { + "url": "https://github.com/kaitai-io/kaitai_struct/issues" + }, + "bundleDependencies": false, + "deprecated": false, + "description": "Kaitai Struct Compiler", + "homepage": "https://github.com/kaitai-io/kaitai_struct_compiler#readme", + "keywords": [ + "kaitai", + "struct", + "compiler", + "binary", + "parsing", + "stream", + "runtime", + "file", + "format", + "structure", + "forenics", + "reversing", + "reverse-engineering" + ], + "license": "GPL-3.0", + "main": "kaitai-struct-compiler.js", + "name": "kaitai-struct-compiler", + "repository": { + "type": "git", + "url": "git+https://github.com/kaitai-io/kaitai_struct_compiler.git" + }, + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "version": "0.8.0-SNAPSHOT.3" +} diff --git a/js/src/main/scala/io/kaitai/struct/MainJs.scala b/js/src/main/scala/io/kaitai/struct/MainJs.scala index 3dffe7d04..a89470b98 100644 --- a/js/src/main/scala/io/kaitai/struct/MainJs.scala +++ b/js/src/main/scala/io/kaitai/struct/MainJs.scala @@ -1,30 +1,29 @@ package io.kaitai.struct -import io.kaitai.struct.format.{ClassSpec, JavaScriptClassSpecs, JavaScriptKSYParser, KSVersion} +import io.kaitai.struct.format.{JavaScriptKSYParser, KSVersion} import io.kaitai.struct.languages.components.LanguageCompilerStatic +import scala.concurrent.ExecutionContext.Implicits.global +import scala.concurrent.Future import scala.scalajs.js import scala.scalajs.js.JSConverters._ import scala.scalajs.js.annotation.JSExport -import scala.concurrent.ExecutionContext.Implicits.global -import scala.concurrent.Future @JSExport object MainJs { - KSVersion.current = BuildInfo.version + KSVersion.current = Version.version @JSExport def compile(langStr: String, yaml: js.Object, importer: JavaScriptImporter, debug: Boolean = false): js.Promise[js.Dictionary[String]] = { try { - val config = new RuntimeConfig(debug = debug) + // TODO: add proper enabled by a flag + //Log.initFromVerboseFlag(Seq("file", "value", "parent", "type_resolve", "type_valid", "seq_sizes", "import")) + val config = new RuntimeConfig(autoRead = !debug, readStoresPos = debug) val lang = LanguageCompilerStatic.byString(langStr) - val yamlScala = JavaScriptKSYParser.yamlJavascriptToScala(yaml) - val firstSpec = ClassSpec.fromYaml(yamlScala) - val specs = new JavaScriptClassSpecs(importer, firstSpec) - Main.importAndPrecompile(specs, config).map { (_) => + JavaScriptKSYParser.yamlToSpecs(yaml, importer, config).map { (specs) => specs.flatMap({ case (_, spec) => - val files = Main.compile(spec, lang, config).files + val files = Main.compile(specs, spec, lang, config).files files.map((x) => x.fileName -> x.contents).toMap }).toJSDictionary }.toJSPromise @@ -37,8 +36,8 @@ object MainJs { lazy val languages: js.Array[String] = LanguageCompilerStatic.NAME_TO_CLASS.keys.toSeq.sorted.toJSArray @JSExport - lazy val version = BuildInfo.version + lazy val version = Version.version @JSExport - lazy val buildDate = BuildInfo.builtAtString + lazy val buildDate = Version.gitTime } diff --git a/js/src/main/scala/io/kaitai/struct/format/JavaScriptKSYParser.scala b/js/src/main/scala/io/kaitai/struct/format/JavaScriptKSYParser.scala index 4aa0e1f29..4aafd4e4d 100644 --- a/js/src/main/scala/io/kaitai/struct/format/JavaScriptKSYParser.scala +++ b/js/src/main/scala/io/kaitai/struct/format/JavaScriptKSYParser.scala @@ -1,8 +1,25 @@ package io.kaitai.struct.format +import io.kaitai.struct.{JavaScriptImporter, Main, RuntimeConfig} + +import scala.concurrent.Future import scala.scalajs.js +import scala.concurrent.ExecutionContext.Implicits.global object JavaScriptKSYParser { + /** + * Converts first YAML (given as JavaScript object) to the ClassSpecs + * object, fully imported and precompiled. + * @param yaml first KSY file (YAML), given as JavaScript object + * @return future of ClassSpecs object + */ + def yamlToSpecs(yaml: Any, importer: JavaScriptImporter, config: RuntimeConfig): Future[ClassSpecs] = { + val yamlScala = yamlJavascriptToScala(yaml) + val firstSpec = ClassSpec.fromYaml(yamlScala) + val specs = new JavaScriptClassSpecs(importer, firstSpec) + Main.importAndPrecompile(specs, config).map((_) => specs) + } + def yamlJavascriptToScala(src: Any): Any = { src match { case array: js.Array[AnyRef] => diff --git a/jvm/src/main/scala/io/kaitai/struct/JavaMain.scala b/jvm/src/main/scala/io/kaitai/struct/JavaMain.scala index 720b9d8ce..262ca3c44 100644 --- a/jvm/src/main/scala/io/kaitai/struct/JavaMain.scala +++ b/jvm/src/main/scala/io/kaitai/struct/JavaMain.scala @@ -7,10 +7,12 @@ import io.kaitai.struct.CompileLog._ import io.kaitai.struct.JavaMain.CLIConfig import io.kaitai.struct.format.{ClassSpec, ClassSpecs, KSVersion, YAMLParseException} import io.kaitai.struct.formats.JavaKSYParser +import io.kaitai.struct.languages.CppCompiler import io.kaitai.struct.languages.components.LanguageCompilerStatic +import io.kaitai.struct.precompile.ErrorInInput object JavaMain { - KSVersion.current = BuildInfo.version + KSVersion.current = Version.version case class CLIConfig( verbose: Seq[String] = Seq(), @@ -23,14 +25,15 @@ object JavaMain { runtime: RuntimeConfig = RuntimeConfig() ) - val ALL_LANGS = LanguageCompilerStatic.NAME_TO_CLASS.keySet - val VALID_LANGS = ALL_LANGS + "all" + val ALL_LANGS = LanguageCompilerStatic.NAME_TO_CLASS.keySet - "cpp_stl" + "cpp_stl_98" + "cpp_stl_11" + val VALID_LANGS = LanguageCompilerStatic.NAME_TO_CLASS.keySet + "all" + val CPP_STANDARDS = Set("98", "11") def parseCommandLine(args: Array[String]): Option[CLIConfig] = { - val parser = new scopt.OptionParser[CLIConfig](BuildInfo.name) { + val parser = new scopt.OptionParser[CLIConfig](Version.name) { override def showUsageOnError = true - head(BuildInfo.name, BuildInfo.version) + head(Version.name, Version.version) arg[File]("...") unbounded() action { (x, c) => c.copy(srcFiles = c.srcFiles :+ x) } text("source files (.ksy)") @@ -50,7 +53,7 @@ object JavaMain { if (VALID_LANGS.contains(x)) { success } else { - failure(s"'${x}' is not a valid target language; valid ones are: ${VALID_LANGS.mkString(", ")}") + failure(s"'$x' is not a valid target language; valid ones are: ${VALID_LANGS.mkString(", ")}") } } @@ -63,14 +66,49 @@ object JavaMain { } text("output directory (filenames will be auto-generated)") val importPathExample = List("", "", "...").mkString(File.pathSeparator) - opt[String]('I', "import-path") valueName(importPathExample) action { (x, c) => + opt[String]('I', "import-path") optional() unbounded() valueName(importPathExample) action { (x, c) => c.copy(importPaths = c.importPaths ++ x.split(File.pathSeparatorChar)) } text(".ksy library search path(s) for imports (see also KSPATH env variable)") + opt[String]("cpp-namespace") valueName("") action { (x, c) => + c.copy( + runtime = c.runtime.copy( + cppConfig = c.runtime.cppConfig.copy( + namespace = x.split("::").toList + ) + ) + ) + } text("C++ namespace (C++ only, default: none)") + + opt[String]("cpp-standard") valueName("") action { (x, c) => + c.copy( + runtime = c.runtime.copy( + cppConfig = x match { + case "98" => c.runtime.cppConfig.copyAsCpp98() + case "11" => c.runtime.cppConfig.copyAsCpp11() + } + ) + ) + } text("C++ standard to target (C++ only, supported: 98, 11, default: 98)") validate { x => + if (CPP_STANDARDS.contains(x)) { + success + } else { + failure(s"'$x' is not a valid C++ standard to target; valid ones are: ${CPP_STANDARDS.mkString(", ")}") + } + } + + opt[String]("go-package") valueName("") action { (x, c) => + c.copy(runtime = c.runtime.copy(goPackage = x)) + } text("Go package (Go only, default: none)") + opt[String]("java-package") valueName("") action { (x, c) => c.copy(runtime = c.runtime.copy(javaPackage = x)) } text("Java package (Java only, default: root package)") + opt[String]("java-from-file-class") valueName("") action { (x, c) => + c.copy(runtime = c.runtime.copy(javaFromFileClass = x)) + } text(s"Java class to be invoked in fromFile() helper (default: ${RuntimeConfig().javaFromFileClass})") + opt[String]("dotnet-namespace") valueName("") action { (x, c) => c.copy(runtime = c.runtime.copy(dotNetNamespace = x)) } text(".NET Namespace (.NET only, default: Kaitai)") @@ -79,6 +117,14 @@ object JavaMain { c.copy(runtime = c.runtime.copy(phpNamespace = x)) } text("PHP Namespace (PHP only, default: root package)") + opt[String]("python-package") valueName("") action { (x, c) => + c.copy(runtime = c.runtime.copy(pythonPackage = x)) + } text("Python package (Python only, default: root package)") + + opt[String]("nim-module") valueName("") action { (x, c) => + c.copy(runtime = c.runtime.copy(nimModule = x)) + } text("Path of Nim runtime module (Nim only, default: kaitai_struct/runtime/nim/kaitai)") + opt[Boolean]("opaque-types") action { (x, c) => c.copy(runtime = c.runtime.copy(opaqueTypes = x)) } text("opaque types allowed, default: false") @@ -106,9 +152,18 @@ object JavaMain { } } + opt[Unit]("no-auto-read") action { (x, c) => + c.copy(runtime = c.runtime.copy(autoRead = false)) + } text("disable auto-running `_read` in constructor") + + opt[Unit]("read-pos") action { (x, c) => + c.copy(runtime = c.runtime.copy(readStoresPos = true)) + } text("`_read` remembers attribute positions in stream") + opt[Unit]("debug") action { (x, c) => - c.copy(runtime = c.runtime.copy(debug = true)) - } text("enable debugging helpers (mostly used by visualization tools)") + c.copy(runtime = c.runtime.copy(autoRead = false, readStoresPos = true)) + } text("same as --no-auto-read --read-pos (useful for visualization tools)") + help("help") text("display this help and exit") version("version") text("output version information and exit") } @@ -211,10 +266,18 @@ class JavaMain(config: CLIConfig) { } srcFile.toString -> log }.toMap - if (config.jsonOutput) + + if (config.jsonOutput) { Console.println(JSON.mapToJson(logs)) + } else { + if (logsHaveErrors(logs)) + System.exit(2) + } } + private def logsHaveErrors(logs: Map[String, InputEntry]): Boolean = + logs.values.map(_.hasErrors).max + private def compileOneInput(srcFile: String) = { Log.fileOps.info(() => s"parsing $srcFile...") val specs = JavaKSYParser.localFileToSpecs(srcFile, config) @@ -242,10 +305,19 @@ class JavaMain(config: CLIConfig) { def compileOneLang(specs: ClassSpecs, langStr: String, outDir: String): Map[String, SpecEntry] = { Log.fileOps.info(() => s"... compiling it for $langStr... ") - val lang = LanguageCompilerStatic.byString(langStr) + + val (lang, fixedRuntime) = langStr match { + case "cpp_stl_98" => + (CppCompiler, config.runtime.copy(cppConfig = config.runtime.cppConfig.copyAsCpp98())) + case "cpp_stl_11" => + (CppCompiler, config.runtime.copy(cppConfig = config.runtime.cppConfig.copyAsCpp11())) + case _ => + (LanguageCompilerStatic.byString(langStr), config.runtime) + } + specs.map { case (_, classSpec) => val res = try { - compileSpecAndWriteToFile(classSpec, lang, outDir) + compileSpecAndWriteToFile(specs, classSpec, lang, fixedRuntime, outDir) } catch { case ex: Throwable => if (config.throwExceptions) @@ -257,11 +329,13 @@ class JavaMain(config: CLIConfig) { } def compileSpecAndWriteToFile( + specs: ClassSpecs, spec: ClassSpec, lang: LanguageCompilerStatic, + runtime: RuntimeConfig, outDir: String ): SpecSuccess = { - val res = Main.compile(spec, lang, config.runtime) + val res = Main.compile(specs, spec, lang, runtime) res.files.foreach { (file) => Log.fileOps.info(() => s".... writing ${file.fileName}") @@ -280,10 +354,17 @@ class JavaMain(config: CLIConfig) { private def exceptionToCompileError(ex: Throwable, srcFile: String): CompileError = { if (!config.jsonOutput) - Console.println(ex.getMessage) + Console.err.println(ex.getMessage) ex match { case ype: YAMLParseException => CompileError("(main)", ype.path, ype.msg) + case e: ErrorInInput => + val file = e.file.getOrElse(srcFile) + val msg = Option(e.getCause) match { + case Some(cause) => cause.getMessage + case None => e.getMessage + } + CompileError(file, e.path, msg) case _ => CompileError(srcFile, List(), ex.getMessage) } diff --git a/jvm/src/main/scala/io/kaitai/struct/formats/JavaKSYParser.scala b/jvm/src/main/scala/io/kaitai/struct/formats/JavaKSYParser.scala index 4a4c33f0b..b6326b20e 100644 --- a/jvm/src/main/scala/io/kaitai/struct/formats/JavaKSYParser.scala +++ b/jvm/src/main/scala/io/kaitai/struct/formats/JavaKSYParser.scala @@ -1,6 +1,7 @@ package io.kaitai.struct.formats -import java.io.{File, FileReader} +import java.io._ +import java.nio.charset.Charset import java.util.{List => JList, Map => JMap} import io.kaitai.struct.JavaMain.CLIConfig @@ -25,11 +26,19 @@ object JavaKSYParser { def fileNameToSpec(yamlFilename: String): ClassSpec = { Log.fileOps.info(() => s"reading $yamlFilename...") - val scalaSrc = readerToYaml(new FileReader(yamlFilename)) + + // This complex string of classes is due to the fact that Java's + // default "FileReader" implementation always uses system locale, + // which screws up encoding on some systems and screws up reading + // UTF-8 files with BOM + val fis = new FileInputStream(yamlFilename) + val isr = new InputStreamReader(fis, Charset.forName("UTF-8")) + val br = new BufferedReader(isr) + val scalaSrc = readerToYaml(br) ClassSpec.fromYaml(scalaSrc) } - def readerToYaml(reader: FileReader): Any = { + def readerToYaml(reader: Reader): Any = { val yamlLoader = new Yaml(new SafeConstructor) val javaSrc = yamlLoader.load(reader) yamlJavaToScala(javaSrc) @@ -55,6 +64,10 @@ object JavaKSYParser { src case javaInt: java.lang.Integer => javaInt.intValue + case javaLong: java.lang.Long => + javaLong.longValue + case _: java.math.BigInteger => + src.toString case null => // may be not the very best idea, but these nulls // should be handled by real parsing code, i.e. where diff --git a/jvm/src/test/scala/io/kaitai/struct/ErrorMessagesSpec.scala b/jvm/src/test/scala/io/kaitai/struct/ErrorMessagesSpec.scala index 64070df45..aad726bf0 100644 --- a/jvm/src/test/scala/io/kaitai/struct/ErrorMessagesSpec.scala +++ b/jvm/src/test/scala/io/kaitai/struct/ErrorMessagesSpec.scala @@ -12,7 +12,7 @@ import org.scalatest.FunSuite class ErrorMessagesSpec extends FunSuite { // required, because this class is the sole entry point and this test needs // version info - KSVersion.current = BuildInfo.version + KSVersion.current = Version.version val FORMATS_ERR_DIR = "../tests/formats_err" val CHARSET_UTF8 = Charset.forName("UTF-8") diff --git a/jvm/src/test/scala/io/kaitai/struct/datatype/SwitchType$Test.scala b/jvm/src/test/scala/io/kaitai/struct/datatype/SwitchType$Test.scala new file mode 100644 index 000000000..eb2fef40f --- /dev/null +++ b/jvm/src/test/scala/io/kaitai/struct/datatype/SwitchType$Test.scala @@ -0,0 +1,40 @@ +package io.kaitai.struct.datatype + +import io.kaitai.struct.datatype.DataType.SwitchType +import io.kaitai.struct.exprlang.Expressions +import io.kaitai.struct.format.ClassSpec +import org.scalatest.FunSpec +import org.scalatest.Matchers._ + +class SwitchType$Test extends FunSpec { + describe("SwitchType.parseSwitch") { + it ("combines ints properly") { + val t = SwitchType( + Expressions.parse("foo"), + Map( + Expressions.parse("1") -> DataType.IntMultiType(true, DataType.Width2, Some(LittleEndian)), + Expressions.parse("2") -> DataType.IntMultiType(false, DataType.Width4, Some(LittleEndian)) + ) + ) + + t.combinedType should be(DataType.CalcIntType) + } + + it ("combines owning user types properly") { + val ut1 = DataType.UserTypeInstream(List("foo"), None) + ut1.classSpec = Some(ClassSpec.opaquePlaceholder(List("foo"))) + val ut2 = DataType.UserTypeInstream(List("bar"), None) +// ut2.classSpec = Some(ClassSpec.opaquePlaceholder(List("bar"))) + + val t = SwitchType( + Expressions.parse("foo"), + Map( + Expressions.parse("1") -> ut1, + Expressions.parse("2") -> ut2 + ) + ) + + t.combinedType should be(DataType.KaitaiStructType) + } + } +} diff --git a/jvm/src/test/scala/io/kaitai/struct/exprlang/Ast$Test.scala b/jvm/src/test/scala/io/kaitai/struct/exprlang/Ast$Test.scala new file mode 100644 index 000000000..28a69b522 --- /dev/null +++ b/jvm/src/test/scala/io/kaitai/struct/exprlang/Ast$Test.scala @@ -0,0 +1,50 @@ +package io.kaitai.struct.exprlang + +import org.scalatest.FunSpec +import org.scalatest.Matchers._ + +class Ast$Test extends FunSpec { + describe("Ast.expr.evaluateIntConst") { + it ("considers `42` constant") { + Expressions.parse("42").evaluateIntConst should be(Some(42)) + } + + it ("considers `-1` constant") { + Expressions.parse("-1").evaluateIntConst should be(Some(-1)) + } + + it ("considers `42 - 2` constant") { + Expressions.parse("42 - 2").evaluateIntConst should be(Some(40)) + } + + it ("considers `(-3 + 7) * 8 / 2` constant") { + Expressions.parse("(-3 + 7) * 8 / 2").evaluateIntConst should be(Some(16)) + } + + it ("considers `[3, 1, 4][2]` constant") { + Expressions.parse("[3, 1, 4][2]").evaluateIntConst should be(Some(4)) + } + + it ("considers `4 > 2 ? 1 : 5` constant") { + Expressions.parse("4 > 2 ? 1 : 5").evaluateIntConst should be(Some(1)) + } + + it ("considers `x` variable") { + Expressions.parse("x").evaluateIntConst should be(None) + } + + it ("considers `[3, 1, 4][x]` variable") { + Expressions.parse("[3, 1, 4][x]").evaluateIntConst should be(None) + } + + // The following tests probably can be improved by some point to deliver + // constant result by introduction of more complex symbolic execution engine. + it ("considers `x - x` non-constant") { + Expressions.parse("x - x").evaluateIntConst should be(None) // be(Some(0)) + } + + it ("considers `(x + 1) * (x + 1) - (x * x + 2 * x + 2)` non-constant") { + Expressions.parse("(x + 1) * (x + 1) - (x * x + 2 * x + 2)").evaluateIntConst should be(None) // be(Some(0)) + } + } +} diff --git a/jvm/src/test/scala/io/kaitai/struct/exprlang/ExpressionsSpec.scala b/jvm/src/test/scala/io/kaitai/struct/exprlang/ExpressionsSpec.scala index 2c07db612..b8382f47d 100644 --- a/jvm/src/test/scala/io/kaitai/struct/exprlang/ExpressionsSpec.scala +++ b/jvm/src/test/scala/io/kaitai/struct/exprlang/ExpressionsSpec.scala @@ -48,6 +48,34 @@ class ExpressionsSpec extends FunSpec { Expressions.parse("0b1010_1_010") should be (IntNum(0xaa)) } + it("parses simple float") { + Expressions.parse("1.2345") should be (FloatNum(1.2345)) + } + + it("parses float with positive exponent") { + Expressions.parse("123e4") should be (FloatNum(123e4)) + } + + it("parses float with positive exponent with plus sign") { + Expressions.parse("123e+4") should be (FloatNum(123e4)) + } + + it("parses float with negative exponent") { + Expressions.parse("123e-7") should be (FloatNum(123e-7)) + } + + it("parses float + non-integral part with positive exponent") { + Expressions.parse("1.2345e7") should be (FloatNum(1.2345e7)) + } + + it("parses float + non-integral part with positive exponent with plus sign") { + Expressions.parse("123.45e+7") should be (FloatNum(123.45e7)) + } + + it("parses float + non-integral part with negative exponent") { + Expressions.parse("123.45e-7") should be (FloatNum(123.45e-7)) + } + it("parses 1 + 2") { Expressions.parse("1 + 2") should be (BinOp(IntNum(1), Add, IntNum(2))) } @@ -103,10 +131,41 @@ class ExpressionsSpec extends FunSpec { Expressions.parse("~(7+3)") should be (UnaryOp(Invert, BinOp(IntNum(7), Add, IntNum(3)))) } + // Enums it("parses port::http") { Expressions.parse("port::http") should be (EnumByLabel(identifier("port"), identifier("http"))) } + it("parses some_type::port::http") { + Expressions.parse("some_type::port::http") should be ( + EnumByLabel( + identifier("port"), + identifier("http"), + typeId(absolute = false, Seq("some_type")) + ) + ) + } + + it("parses parent_type::child_type::port::http") { + Expressions.parse("parent_type::child_type::port::http") should be ( + EnumByLabel( + identifier("port"), + identifier("http"), + typeId(absolute = false, Seq("parent_type", "child_type")) + ) + ) + } + + it("parses ::parent_type::child_type::port::http") { + Expressions.parse("::parent_type::child_type::port::http") should be ( + EnumByLabel( + identifier("port"), + identifier("http"), + typeId(absolute = true, Seq("parent_type", "child_type")) + ) + ) + } + it("parses port::http.to_i + 8000 == 8080") { Expressions.parse("port::http.to_i + 8000 == 8080") should be ( Compare( @@ -143,6 +202,36 @@ class ExpressionsSpec extends FunSpec { Expressions.parse("truer") should be (Name(identifier("truer"))) } + // Boolean operations + it("parses not foo") { + Expressions.parse("not foo") should be ( + UnaryOp( + Ast.unaryop.Not, + Name(identifier("foo")) + ) + ) + } + + it("parses note_len") { + Expressions.parse("note_len") should be (Name(identifier("note_len"))) + } + + it("parses notnot") { + Expressions.parse("notnot") should be (Name(identifier("notnot"))) + } + + it("parses not not true") { + Expressions.parse("not not true") should be ( + UnaryOp( + Ast.unaryop.Not, + UnaryOp( + Ast.unaryop.Not, + Bool(true) + ) + ) + ) + } + // String literals it("parses simple string") { Expressions.parse("\"abc\"") should be (Str("abc")) @@ -174,23 +263,57 @@ class ExpressionsSpec extends FunSpec { // Casts it("parses 123.as") { - Expressions.parse("123.as") should be (CastToType(IntNum(123),identifier("u4"))) + Expressions.parse("123.as") should be ( + CastToType(IntNum(123), typeId(false, Seq("u4"))) + ) } it("parses (123).as") { - Expressions.parse("(123).as") should be (CastToType(IntNum(123),identifier("u4"))) + Expressions.parse("(123).as") should be ( + CastToType(IntNum(123), typeId(false, Seq("u4"))) + ) } it("parses \"str\".as") { - Expressions.parse("\"str\".as") should be (CastToType(Str("str"),identifier("x"))) + Expressions.parse("\"str\".as") should be ( + CastToType(Str("str"), typeId(false, Seq("x"))) + ) } it("parses foo.as") { - Expressions.parse("foo.as") should be (CastToType(Name(identifier("foo")),identifier("x"))) + Expressions.parse("foo.as") should be ( + CastToType(Name(identifier("foo")), typeId(false, Seq("x"))) + ) } it("parses foo.as < x > ") { - Expressions.parse("foo.as < x > ") should be (CastToType(Name(identifier("foo")),identifier("x"))) + Expressions.parse("foo.as < x > ") should be ( + CastToType(Name(identifier("foo")), typeId(false, Seq("x"))) + ) + } + + it("parses foo.as") { + Expressions.parse("foo.as") should be ( + CastToType(Name(identifier("foo")), typeId(false, Seq("bar", "baz"))) + ) + } + + it("parses foo.as<::bar::baz>") { + Expressions.parse("foo.as<::bar::baz>") should be ( + CastToType(Name(identifier("foo")), typeId(true, Seq("bar", "baz"))) + ) + } + + it("parses foo.as") { + Expressions.parse("foo.as") should be ( + CastToType(Name(identifier("foo")), typeId(false, Seq("bar"), true)) + ) + } + + it("parses foo.as<::bar::baz[]>") { + Expressions.parse("foo.as<::bar::baz[]>") should be ( + CastToType(Name(identifier("foo")), typeId(true, Seq("bar", "baz"), true)) + ) } it("parses foo.as") { @@ -207,6 +330,52 @@ class ExpressionsSpec extends FunSpec { ) } + // sizeof keyword + it("parses sizeof") { + Expressions.parse("sizeof") should be ( + ByteSizeOfType(typeId(false, Seq("foo"))) + ) + } + + it("parses sizeof") { + Expressions.parse("sizeof") should be ( + ByteSizeOfType(typeId(false, Seq("foo", "bar"))) + ) + } + + it("parses sizeof<::foo::bar>") { + Expressions.parse("sizeof<::foo::bar>") should be ( + ByteSizeOfType(typeId(true, Seq("foo", "bar"))) + ) + } + + it("parses sizeof") { + Expressions.parse("bitsizeof") should be ( + BitSizeOfType(typeId(false, Seq("foo"))) + ) + } + + it("parses bitsizeof "100000000000L" - )), - - // Float literals - everybody("1.0", "1.0", CalcFloatType), - everybody("123.456", "123.456", CalcFloatType), - everybody("-123.456", "-123.456", CalcFloatType), - - // Simple integer operations - everybody("1 + 2", "(1 + 2)"), - - everybodyExcept("3 / 2", "(3 / 2)", Map( - JavaScriptCompiler -> "Math.floor(3 / 2)", - PerlCompiler -> "int(3 / 2)", - PHPCompiler -> "intval(3 / 2)", - PythonCompiler -> "3 // 2" - )), - - everybody("1 + 2 + 5", "((1 + 2) + 5)"), - - everybodyExcept("(1 + 2) / (7 * 8)", "((1 + 2) / (7 * 8))", Map( - JavaScriptCompiler -> "Math.floor((1 + 2) / (7 * 8))", - PerlCompiler -> "int((1 + 2) / (7 * 8))", - PHPCompiler -> "intval((1 + 2) / (7 * 8))", - PythonCompiler -> "(1 + 2) // (7 * 8)" - )), - - everybody("1 < 2", "1 < 2", CalcBooleanType), - - everybody("1 == 2", "1 == 2", CalcBooleanType), - - full("2 < 3 ? \"foo\" : \"bar\"", CalcIntType, CalcStrType, Map[LanguageCompilerStatic, String]( - CppCompiler -> "(2 < 3) ? (std::string(\"foo\")) : (std::string(\"bar\"))", - CSharpCompiler -> "2 < 3 ? \"foo\" : \"bar\"", - JavaCompiler -> "2 < 3 ? \"foo\" : \"bar\"", - JavaScriptCompiler -> "2 < 3 ? \"foo\" : \"bar\"", - PerlCompiler -> "2 < 3 ? \"foo\" : \"bar\"", - PHPCompiler -> "2 < 3 ? \"foo\" : \"bar\"", - PythonCompiler -> "u\"foo\" if 2 < 3 else u\"bar\"", - RubyCompiler -> "2 < 3 ? \"foo\" : \"bar\"" - )), - - everybody("~777", "~777"), - everybody("~(7+3)", "~(7 + 3)"), - - // Simple float operations - everybody("1.2 + 3.4", "(1.2 + 3.4)", CalcFloatType), - everybody("1.2 + 3", "(1.2 + 3)", CalcFloatType), - everybody("1 + 3.4", "(1 + 3.4)", CalcFloatType), - - everybody("1.0 < 2", "1.0 < 2", CalcBooleanType), - - everybody("3 / 2.0", "(3 / 2.0)", CalcFloatType), - - everybody("(1 + 2) / (7 * 8.1)", "((1 + 2) / (7 * 8.1))", CalcFloatType), - - // Boolean literals - full("true", CalcBooleanType, CalcBooleanType, Map[LanguageCompilerStatic, String]( - CppCompiler -> "true", - CSharpCompiler -> "true", - JavaCompiler -> "true", - JavaScriptCompiler -> "true", - PerlCompiler -> "1", - PHPCompiler -> "true", - PythonCompiler -> "True", - RubyCompiler -> "true" - )), - - full("false", CalcBooleanType, CalcBooleanType, Map[LanguageCompilerStatic, String]( - CppCompiler -> "false", - CSharpCompiler -> "false", - JavaCompiler -> "false", - JavaScriptCompiler -> "false", - PerlCompiler -> "0", - PHPCompiler -> "false", - PythonCompiler -> "False", - RubyCompiler -> "false" - )), - - full("some_bool.to_i", CalcBooleanType, CalcIntType, Map[LanguageCompilerStatic, String]( - CppCompiler -> "some_bool()", - CSharpCompiler -> "(SomeBool ? 1 : 0)", - JavaCompiler -> "(someBool() ? 1 : 0)", - JavaScriptCompiler -> "(this.someBool | 0)", - PerlCompiler -> "$self->some_bool()", - PHPCompiler -> "intval($this->someBool())", - PythonCompiler -> "int(self.some_bool)", - RubyCompiler -> "(some_bool ? 1 : 0)" - )), - - // Member access - full("foo_str", CalcStrType, CalcStrType, Map[LanguageCompilerStatic, String]( - CppCompiler -> "foo_str()", - CSharpCompiler -> "FooStr", - JavaCompiler -> "fooStr()", - JavaScriptCompiler -> "this.fooStr", - PerlCompiler -> "$self->foo_str()", - PHPCompiler -> "$this->fooStr()", - PythonCompiler -> "self.foo_str", - RubyCompiler -> "foo_str" - )), - - full("foo_block", userType("block"), userType("block"), Map[LanguageCompilerStatic, String]( - CppCompiler -> "foo_block()", - CSharpCompiler -> "FooBlock", - JavaCompiler -> "fooBlock()", - JavaScriptCompiler -> "this.fooBlock", - PerlCompiler -> "$self->foo_block()", - PHPCompiler -> "$this->fooBlock()", - PythonCompiler -> "self.foo_block", - RubyCompiler -> "foo_block" - )), - - full("foo.bar", FooBarProvider, CalcStrType, Map[LanguageCompilerStatic, String]( - CppCompiler -> "foo()->bar()", - CSharpCompiler -> "Foo.Bar", - JavaCompiler -> "foo().bar()", - JavaScriptCompiler -> "this.foo.bar", - PerlCompiler -> "$self->foo()->bar()", - PHPCompiler -> "$this->foo()->bar()", - PythonCompiler -> "self.foo.bar", - RubyCompiler -> "foo.bar" - )), - - full("foo.inner.baz", FooBarProvider, CalcIntType, Map[LanguageCompilerStatic, String]( - CppCompiler -> "foo()->inner()->baz()", - CSharpCompiler -> "Foo.Inner.Baz", - JavaCompiler -> "foo().inner().baz()", - JavaScriptCompiler -> "this.foo.inner.baz", - PerlCompiler -> "$self->foo()->inner()->baz()", - PHPCompiler -> "$this->foo()->inner()->baz()", - PythonCompiler -> "self.foo.inner.baz", - RubyCompiler -> "foo.inner.baz" - )), - - full("_root.foo", userType("block"), userType("block"), Map[LanguageCompilerStatic, String]( - CppCompiler -> "_root()->foo()", - CSharpCompiler -> "M_Root.Foo", - JavaCompiler -> "_root.foo()", - JavaScriptCompiler -> "this._root.foo", - PerlCompiler -> "$self->_root()->foo()", - PHPCompiler -> "$this->_root()->foo()", - PythonCompiler -> "self._root.foo", - RubyCompiler -> "_root.foo" - )), - - full("a != 2 and a != 5", CalcIntType, CalcBooleanType, Map[LanguageCompilerStatic, String]( - CppCompiler -> "a() != 2 && a() != 5", - CSharpCompiler -> "A != 2 && A != 5", - JavaCompiler -> "a() != 2 && a() != 5", - JavaScriptCompiler -> "this.a != 2 && this.a != 5", - PerlCompiler -> "$self->a() != 2 && $self->a() != 5", - PHPCompiler -> "$this->a() != 2 && $this->a() != 5", - PythonCompiler -> "self.a != 2 and self.a != 5", - RubyCompiler -> "a != 2 && a != 5" - )), - - // Arrays - full("[0, 1, 100500]", CalcIntType, ArrayType(CalcIntType), Map[LanguageCompilerStatic, String]( - CSharpCompiler -> "new List { 0, 1, 100500 }", - JavaCompiler -> "new ArrayList(Arrays.asList(0, 1, 100500))", - JavaScriptCompiler -> "[0, 1, 100500]", - PerlCompiler -> "(0, 1, 100500)", - PHPCompiler -> "[0, 1, 100500]", - PythonCompiler -> "[0, 1, 100500]", - RubyCompiler -> "[0, 1, 100500]" - )), - - full("[34, 0, 10, 64, 65, 66, 92]", CalcIntType, CalcBytesType, Map[LanguageCompilerStatic, String]( - CppCompiler -> "std::string(\"\\x22\\x00\\x0A\\x40\\x41\\x42\\x5C\", 7)", - CSharpCompiler -> "new byte[] { 34, 0, 10, 64, 65, 66, 92 }", - JavaCompiler -> "new byte[] { 34, 0, 10, 64, 65, 66, 92 }", - JavaScriptCompiler -> "[34, 0, 10, 64, 65, 66, 92]", - PerlCompiler -> "pack('C*', (34, 0, 10, 64, 65, 66, 92))", - PHPCompiler -> "\"\\x22\\x00\\x0A\\x40\\x41\\x42\\x5C\"", - PythonCompiler -> "struct.pack('7b', 34, 0, 10, 64, 65, 66, 92)", - RubyCompiler -> "[34, 0, 10, 64, 65, 66, 92].pack('C*')" - )), - - full("[255, 0, 255]", CalcIntType, CalcBytesType, Map[LanguageCompilerStatic, String]( - CppCompiler -> "std::string(\"\\xFF\\x00\\xFF\", 3)", - CSharpCompiler -> "new byte[] { 255, 0, 255 }", - JavaCompiler -> "new byte[] { -1, 0, -1 }", - JavaScriptCompiler -> "[255, 0, 255]", - PerlCompiler -> "pack('C*', (255, 0, 255))", - PHPCompiler -> "\"\\xFF\\x00\\xFF\"", - PythonCompiler -> "struct.pack('3b', -1, 0, -1)", - RubyCompiler -> "[255, 0, 255].pack('C*')" - )), - - full("a[42]", ArrayType(CalcStrType), CalcStrType, Map[LanguageCompilerStatic, String]( - CppCompiler -> "a()->at(42)", - CSharpCompiler -> "A[42]", - JavaCompiler -> "a().get(42)", - JavaScriptCompiler -> "this.a[42]", - PythonCompiler -> "self.a[42]", - RubyCompiler -> "a[42]" - )), - - full("a[42 - 2]", ArrayType(CalcStrType), CalcStrType, Map[LanguageCompilerStatic, String]( - CppCompiler -> "a()->at((42 - 2))", - CSharpCompiler -> "A[(42 - 2)]", - JavaCompiler -> "a().get((42 - 2))", - JavaScriptCompiler -> "this.a[(42 - 2)]", - PythonCompiler -> "self.a[(42 - 2)]", - RubyCompiler -> "a[(42 - 2)]" - )), - - full("a.first", ArrayType(CalcIntType), CalcIntType, Map[LanguageCompilerStatic, String]( - CppCompiler -> "a()->front()", - CSharpCompiler -> "A[0]", - JavaCompiler -> "a().get(0)", - JavaScriptCompiler -> "this.a[0]", - PythonCompiler -> "self.a[0]", - RubyCompiler -> "a.first" - )), - - full("a.last", ArrayType(CalcIntType), CalcIntType, Map[LanguageCompilerStatic, String]( - CppCompiler -> "a()->back()", - CSharpCompiler -> "A[A.Length - 1]", - JavaCompiler -> "a().get(a().size() - 1)", - JavaScriptCompiler -> "this.a[this.a.length - 1]", - PythonCompiler -> "self.a[-1]", - RubyCompiler -> "a.last" - )), - - full("a.size", ArrayType(CalcIntType), CalcIntType, Map[LanguageCompilerStatic, String]( - CppCompiler -> "a()->size()", - CSharpCompiler -> "A.Count", - JavaCompiler -> "a().size()", - JavaScriptCompiler -> "this.a.length", - PHPCompiler -> "count(a)", - PerlCompiler -> "scalar($self->a())", - PythonCompiler -> "len(self.a)", - RubyCompiler -> "a.length" - )), - - // Strings - full("\"str\"", CalcIntType, CalcStrType, Map[LanguageCompilerStatic, String]( - CppCompiler -> "std::string(\"str\")", - CSharpCompiler -> "\"str\"", - JavaCompiler -> "\"str\"", - JavaScriptCompiler -> "\"str\"", - PerlCompiler -> "\"str\"", - PHPCompiler -> "\"str\"", - PythonCompiler -> "u\"str\"", - RubyCompiler -> "\"str\"" - )), - - full("\"str\\nnext\"", CalcIntType, CalcStrType, Map[LanguageCompilerStatic, String]( - CppCompiler -> "std::string(\"str\\nnext\")", - CSharpCompiler -> "\"str\\nnext\"", - JavaCompiler -> "\"str\\nnext\"", - JavaScriptCompiler -> "\"str\\nnext\"", - PerlCompiler -> "\"str\\nnext\"", - PHPCompiler -> "\"str\\nnext\"", - PythonCompiler -> "u\"str\\nnext\"", - RubyCompiler -> "\"str\\nnext\"" - )), - - full("\"str\\u000anext\"", CalcIntType, CalcStrType, Map[LanguageCompilerStatic, String]( - CppCompiler -> "std::string(\"str\\nnext\")", - CSharpCompiler -> "\"str\\nnext\"", - JavaCompiler -> "\"str\\nnext\"", - JavaScriptCompiler -> "\"str\\nnext\"", - PerlCompiler -> "\"str\\nnext\"", - PHPCompiler -> "\"str\\nnext\"", - PythonCompiler -> "u\"str\\nnext\"", - RubyCompiler -> "\"str\\nnext\"" - )), - - full("\"str\\0next\"", CalcIntType, CalcStrType, Map[LanguageCompilerStatic, String]( - CppCompiler -> "std::string(\"str\\000next\", 8)", - CSharpCompiler -> "\"str\\0next\"", - JavaCompiler -> "\"str\\000next\"", - JavaScriptCompiler -> "\"str\\000next\"", - PerlCompiler -> "\"str\\000next\"", - PHPCompiler -> "\"str\\000next\"", - PythonCompiler -> "u\"str\\000next\"", - RubyCompiler -> "\"str\\000next\"" - )), - - everybodyExcept("\"str1\" + \"str2\"", "\"str1\" + \"str2\"", Map[LanguageCompilerStatic, String]( - CppCompiler -> "std::string(\"str1\") + std::string(\"str2\")", - PerlCompiler -> "\"str1\" . \"str2\"", - PHPCompiler -> "\"str1\" . \"str2\"", - PythonCompiler -> "u\"str1\" + u\"str2\"" - ), CalcStrType), - - everybodyExcept("\"str1\" == \"str2\"", "\"str1\" == \"str2\"", Map[LanguageCompilerStatic, String]( - CppCompiler -> "std::string(\"str1\") == (std::string(\"str2\"))", - JavaCompiler -> "\"str1\".equals(\"str2\")", - PerlCompiler -> "\"str1\" eq \"str2\"", - PythonCompiler -> "u\"str1\" == u\"str2\"" - ), CalcBooleanType), - - everybodyExcept("\"str1\" != \"str2\"", "\"str1\" != \"str2\"", Map[LanguageCompilerStatic, String]( - CppCompiler -> "std::string(\"str1\") != std::string(\"str2\")", - JavaCompiler -> "!(\"str1\").equals(\"str2\")", - PerlCompiler -> "\"str1\" ne \"str2\"", - PythonCompiler -> "u\"str1\" != u\"str2\"" - ), CalcBooleanType), - - everybodyExcept("\"str1\" < \"str2\"", "\"str1\" < \"str2\"", Map[LanguageCompilerStatic, String]( - CppCompiler -> "(std::string(\"str1\").compare(std::string(\"str2\")) < 0)", - CSharpCompiler -> "(\"str1\".CompareTo(\"str2\") < 0)", - JavaCompiler -> "(\"str1\".compareTo(\"str2\") < 0)", - PerlCompiler -> "\"str1\" lt \"str2\"", - PythonCompiler -> "u\"str1\" < u\"str2\"" - ), CalcBooleanType), - - full("\"str\".length", CalcIntType, CalcIntType, Map[LanguageCompilerStatic, String]( - CppCompiler -> "std::string(\"str\").length()", - CSharpCompiler -> "\"str\".Length", - JavaCompiler -> "\"str\".length()", - JavaScriptCompiler -> "\"str\".length", - PerlCompiler -> "length(\"str\")", - PHPCompiler -> "strlen(\"str\")", - PythonCompiler -> "len(u\"str\")", - RubyCompiler -> "\"str\".size" - )), - - full("\"str\".reverse", CalcIntType, CalcStrType, Map[LanguageCompilerStatic, String]( - CppCompiler -> "kaitai::kstream::reverse(std::string(\"str\"))", - CSharpCompiler -> "new string(Array.Reverse(\"str\".ToCharArray()))", - JavaCompiler -> "new StringBuilder(\"str\").reverse().toString()", - JavaScriptCompiler -> "Array.from(\"str\").reverse().join('')", - PerlCompiler -> "scalar(reverse(\"str\"))", - PHPCompiler -> "strrev(\"str\")", - PythonCompiler -> "u\"str\"[::-1]", - RubyCompiler -> "\"str\".reverse" - )), - - full("\"12345\".to_i", CalcIntType, CalcIntType, Map[LanguageCompilerStatic, String]( - CppCompiler -> "std::stoi(std::string(\"12345\"))", - CSharpCompiler -> "Convert.ToInt64(\"12345\", 10)", - JavaCompiler -> "Long.parseLong(\"12345\", 10)", - JavaScriptCompiler -> "Number.parseInt(\"12345\", 10)", - PerlCompiler -> "\"12345\"", - PHPCompiler -> "intval(\"12345\", 10)", - PythonCompiler -> "int(u\"12345\")", - RubyCompiler -> "\"12345\".to_i" - )), - - full("\"1234fe\".to_i(16)", CalcIntType, CalcIntType, Map[LanguageCompilerStatic, String]( - CppCompiler -> "std::stoi(std::string(\"1234fe\"), 0, 16)", - CSharpCompiler -> "Convert.ToInt64(\"1234fe\", 16)", - JavaCompiler -> "Long.parseLong(\"1234fe\", 16)", - JavaScriptCompiler -> "Number.parseInt(\"1234fe\", 16)", - PerlCompiler -> "hex(\"1234fe\")", - PHPCompiler -> "intval(\"1234fe\", 16)", - PythonCompiler -> "int(u\"1234fe\", 16)", - RubyCompiler -> "\"1234fe\".to_i(16)" - )), - - // casts - full("other.as.bar", FooBarProvider, CalcStrType, Map[LanguageCompilerStatic, String]( - CppCompiler -> "static_cast(other())->bar()", - CSharpCompiler -> "((Block) (Other)).Bar", - JavaCompiler -> "((Block) (other())).bar()", - JavaScriptCompiler -> "this.other.bar", - PerlCompiler -> "$self->other()->bar()", - PHPCompiler -> "$this->other()->bar()", - PythonCompiler -> "self.other.bar", - RubyCompiler -> "other.bar" - )), - - // very simple workaround for Scala not having optional trailing commas - everybody("999", "999") - ) - - for ((src, tp, expType, expOut) <- tests) { + +class TranslatorSpec extends FunSuite { + + // Integer literals + unary minus + everybody("123", "123", Int1Type(true)) + everybody("223", "223", Int1Type(false)) + everybody("1234", "1234") + everybody("-456", "-456") + everybody("0x1234", "4660") + // less and more than 32 Bit signed int + everybody("1000000000", "1000000000") + everybodyExcept("100000000000", "100000000000", Map[LanguageCompilerStatic, String]( + JavaCompiler -> "100000000000L" + )) + + // Float literals + everybody("1.0", "1.0", CalcFloatType) + everybody("123.456", "123.456", CalcFloatType) + everybody("-123.456", "-123.456", CalcFloatType) + + // Simple integer operations + everybody("1 + 2", "(1 + 2)") + + everybodyExcept("3 / 2", "(3 / 2)", Map( + JavaScriptCompiler -> "Math.floor(3 / 2)", + PerlCompiler -> "int(3 / 2)", + PHPCompiler -> "intval(3 / 2)", + PythonCompiler -> "3 // 2" + )) + + everybody("1 + 2 + 5", "((1 + 2) + 5)") + + everybodyExcept("(1 + 2) / (7 * 8)", "((1 + 2) / (7 * 8))", Map( + JavaScriptCompiler -> "Math.floor((1 + 2) / (7 * 8))", + PerlCompiler -> "int((1 + 2) / (7 * 8))", + PHPCompiler -> "intval((1 + 2) / (7 * 8))", + PythonCompiler -> "(1 + 2) // (7 * 8)" + )) + + everybody("1 < 2", "1 < 2", CalcBooleanType) + + everybody("1 == 2", "1 == 2", CalcBooleanType) + + full("2 < 3 ? \"foo\" : \"bar\"", CalcIntType, CalcStrType, Map[LanguageCompilerStatic, String]( + CppCompiler -> "(2 < 3) ? (std::string(\"foo\")) : (std::string(\"bar\"))", + CSharpCompiler -> "2 < 3 ? \"foo\" : \"bar\"", + GoCompiler -> """var tmp1 string; + |if (2 < 3) { + | tmp1 = "foo" + |} else { + | tmp1 = "bar" + |} + |tmp1""".stripMargin, + JavaCompiler -> "2 < 3 ? \"foo\" : \"bar\"", + JavaScriptCompiler -> "2 < 3 ? \"foo\" : \"bar\"", + LuaCompiler -> "2 < 3 and \"foo\" or \"bar\"", + PerlCompiler -> "2 < 3 ? \"foo\" : \"bar\"", + PHPCompiler -> "2 < 3 ? \"foo\" : \"bar\"", + PythonCompiler -> "u\"foo\" if 2 < 3 else u\"bar\"", + RubyCompiler -> "2 < 3 ? \"foo\" : \"bar\"" + )) + + everybodyExcept("~777", "~777", Map[LanguageCompilerStatic, String]( + GoCompiler -> "^777" + )) + everybodyExcept("~(7+3)", "~((7 + 3))", Map[LanguageCompilerStatic, String]( + GoCompiler -> "^((7 + 3))" + )) + + // Simple float operations + everybody("1.2 + 3.4", "(1.2 + 3.4)", CalcFloatType) + everybody("1.2 + 3", "(1.2 + 3)", CalcFloatType) + everybody("1 + 3.4", "(1 + 3.4)", CalcFloatType) + + everybody("1.0 < 2", "1.0 < 2", CalcBooleanType) + + everybody("3 / 2.0", "(3 / 2.0)", CalcFloatType) + + everybody("(1 + 2) / (7 * 8.1)", "((1 + 2) / (7 * 8.1))", CalcFloatType) + + // Boolean literals + full("true", CalcBooleanType, CalcBooleanType, Map[LanguageCompilerStatic, String]( + CppCompiler -> "true", + CSharpCompiler -> "true", + GoCompiler -> "true", + JavaCompiler -> "true", + JavaScriptCompiler -> "true", + LuaCompiler -> "true", + PerlCompiler -> "1", + PHPCompiler -> "true", + PythonCompiler -> "True", + RubyCompiler -> "true" + )) + + full("false", CalcBooleanType, CalcBooleanType, Map[LanguageCompilerStatic, String]( + CppCompiler -> "false", + CSharpCompiler -> "false", + GoCompiler -> "false", + JavaCompiler -> "false", + JavaScriptCompiler -> "false", + LuaCompiler -> "false", + PerlCompiler -> "0", + PHPCompiler -> "false", + PythonCompiler -> "False", + RubyCompiler -> "false" + )) + + full("some_bool.to_i", CalcBooleanType, CalcIntType, Map[LanguageCompilerStatic, String]( + CppCompiler -> "some_bool()", + CSharpCompiler -> "(SomeBool ? 1 : 0)", + GoCompiler -> """tmp1 := 0 + |if this.SomeBool { + | tmp1 = 1 + |} + |tmp1""".stripMargin, + JavaCompiler -> "(someBool() ? 1 : 0)", + JavaScriptCompiler -> "(this.someBool | 0)", + LuaCompiler -> "self.some_bool and 1 or 0", + PerlCompiler -> "$self->some_bool()", + PHPCompiler -> "intval($this->someBool())", + PythonCompiler -> "int(self.some_bool)", + RubyCompiler -> "(some_bool ? 1 : 0)" + )) + + // Member access + full("foo_str", CalcStrType, CalcStrType, Map[LanguageCompilerStatic, String]( + CppCompiler -> "foo_str()", + CSharpCompiler -> "FooStr", + GoCompiler -> "this.FooStr", + JavaCompiler -> "fooStr()", + JavaScriptCompiler -> "this.fooStr", + LuaCompiler -> "self.foo_str", + PerlCompiler -> "$self->foo_str()", + PHPCompiler -> "$this->fooStr()", + PythonCompiler -> "self.foo_str", + RubyCompiler -> "foo_str" + )) + + full("foo_block", userType(List("block")), userType(List("block")), Map[LanguageCompilerStatic, String]( + CppCompiler -> "foo_block()", + CSharpCompiler -> "FooBlock", + GoCompiler -> "this.FooBlock", + JavaCompiler -> "fooBlock()", + JavaScriptCompiler -> "this.fooBlock", + LuaCompiler -> "self.foo_block", + PerlCompiler -> "$self->foo_block()", + PHPCompiler -> "$this->fooBlock()", + PythonCompiler -> "self.foo_block", + RubyCompiler -> "foo_block" + )) + + full("foo.bar", FooBarProvider, CalcStrType, Map[LanguageCompilerStatic, String]( + CppCompiler -> "foo()->bar()", + CSharpCompiler -> "Foo.Bar", + GoCompiler -> "this.Foo.Bar", + JavaCompiler -> "foo().bar()", + JavaScriptCompiler -> "this.foo.bar", + LuaCompiler -> "self.foo.bar", + PerlCompiler -> "$self->foo()->bar()", + PHPCompiler -> "$this->foo()->bar()", + PythonCompiler -> "self.foo.bar", + RubyCompiler -> "foo.bar" + )) + + full("foo.inner.baz", FooBarProvider, CalcIntType, Map[LanguageCompilerStatic, String]( + CppCompiler -> "foo()->inner()->baz()", + CSharpCompiler -> "Foo.Inner.Baz", + GoCompiler -> "this.Foo.Inner.Baz", + JavaCompiler -> "foo().inner().baz()", + JavaScriptCompiler -> "this.foo.inner.baz", + LuaCompiler -> "self.foo.inner.baz", + PerlCompiler -> "$self->foo()->inner()->baz()", + PHPCompiler -> "$this->foo()->inner()->baz()", + PythonCompiler -> "self.foo.inner.baz", + RubyCompiler -> "foo.inner.baz" + )) + + full("_root.foo", userType(List("top_class", "block")), userType(List("top_class", "block")), Map[LanguageCompilerStatic, String]( + CppCompiler -> "_root()->foo()", + CSharpCompiler -> "M_Root.Foo", + GoCompiler -> "this._root.Foo", + JavaCompiler -> "_root.foo()", + JavaScriptCompiler -> "this._root.foo", + LuaCompiler -> "self._root.foo", + PerlCompiler -> "$self->_root()->foo()", + PHPCompiler -> "$this->_root()->foo()", + PythonCompiler -> "self._root.foo", + RubyCompiler -> "_root.foo" + )) + + full("a != 2 and a != 5", CalcIntType, CalcBooleanType, Map[LanguageCompilerStatic, String]( + CppCompiler -> "a() != 2 && a() != 5", + CSharpCompiler -> "A != 2 && A != 5", + GoCompiler -> "a != 2 && a != 5", + JavaCompiler -> "a() != 2 && a() != 5", + JavaScriptCompiler -> "this.a != 2 && this.a != 5", + LuaCompiler -> "self.a ~= 2 and self.a ~= 5", + PerlCompiler -> "$self->a() != 2 && $self->a() != 5", + PHPCompiler -> "$this->a() != 2 && $this->a() != 5", + PythonCompiler -> "self.a != 2 and self.a != 5", + RubyCompiler -> "a != 2 && a != 5" + )) + + // Arrays + full("[0, 1, 100500]", CalcIntType, ArrayType(CalcIntType), Map[LanguageCompilerStatic, String]( + CSharpCompiler -> "new List { 0, 1, 100500 }", + GoCompiler -> "[]int{0, 1, 100500}", + JavaCompiler -> "new ArrayList(Arrays.asList(0, 1, 100500))", + JavaScriptCompiler -> "[0, 1, 100500]", + LuaCompiler -> "{0, 1, 100500}", + PerlCompiler -> "(0, 1, 100500)", + PHPCompiler -> "[0, 1, 100500]", + PythonCompiler -> "[0, 1, 100500]", + RubyCompiler -> "[0, 1, 100500]" + )) + + full("[34, 0, 10, 64, 65, 66, 92]", CalcIntType, CalcBytesType, Map[LanguageCompilerStatic, String]( + CppCompiler -> "std::string(\"\\x22\\x00\\x0A\\x40\\x41\\x42\\x5C\", 7)", + CSharpCompiler -> "new byte[] { 34, 0, 10, 64, 65, 66, 92 }", + GoCompiler -> "\"\\x22\\x00\\x0A\\x40\\x41\\x42\\x5C\"", + JavaCompiler -> "new byte[] { 34, 0, 10, 64, 65, 66, 92 }", + JavaScriptCompiler -> "[34, 0, 10, 64, 65, 66, 92]", + LuaCompiler -> "\"\\034\\000\\010\\064\\065\\066\\092\"", + PerlCompiler -> "pack('C*', (34, 0, 10, 64, 65, 66, 92))", + PHPCompiler -> "\"\\x22\\x00\\x0A\\x40\\x41\\x42\\x5C\"", + PythonCompiler -> "b\"\\x22\\x00\\x0A\\x40\\x41\\x42\\x5C\"", + RubyCompiler -> "[34, 0, 10, 64, 65, 66, 92].pack('C*')" + )) + + full("[255, 0, 255]", CalcIntType, CalcBytesType, Map[LanguageCompilerStatic, String]( + CppCompiler -> "std::string(\"\\xFF\\x00\\xFF\", 3)", + CSharpCompiler -> "new byte[] { 255, 0, 255 }", + GoCompiler -> "\"\\xFF\\x00\\xFF\"", + JavaCompiler -> "new byte[] { -1, 0, -1 }", + JavaScriptCompiler -> "[255, 0, 255]", + LuaCompiler -> "\"\\255\\000\\255\"", + PerlCompiler -> "pack('C*', (255, 0, 255))", + PHPCompiler -> "\"\\xFF\\x00\\xFF\"", + PythonCompiler -> "b\"\\255\\000\\255\"", + RubyCompiler -> "[255, 0, 255].pack('C*')" + )) + + full("[0, 1, 2].length", CalcIntType, CalcIntType, Map[LanguageCompilerStatic, String]( + CppCompiler -> "std::string(\"\\x00\\x01\\x02\", 3).length()", + GoCompiler -> "len(\"\\x00\\x01\\x02\")", + JavaCompiler -> "new byte[] { 0, 1, 2 }.length", + LuaCompiler -> "string.len(\"str\")", + PerlCompiler -> "length(pack('C*', (0, 1, 2)))", + PHPCompiler -> "strlen(\"\\x00\\x01\\x02\")", + PythonCompiler -> "len(b\"\\x00\\x01\\x02\")", + RubyCompiler -> "[0, 1, 2].pack('C*').size" + )) + + full("a[42]", ArrayType(CalcStrType), CalcStrType, Map[LanguageCompilerStatic, String]( + CppCompiler -> "a()->at(42)", + CSharpCompiler -> "A[42]", + GoCompiler -> "this.A[42]", + JavaCompiler -> "a().get((int) 42)", + JavaScriptCompiler -> "this.a[42]", + LuaCompiler -> "self.a[43]", + PHPCompiler -> "$this->a()[42]", + PythonCompiler -> "self.a[42]", + RubyCompiler -> "a[42]" + )) + + full("a[42 - 2]", ArrayType(CalcStrType), CalcStrType, Map[LanguageCompilerStatic, String]( + CppCompiler -> "a()->at((42 - 2))", + CSharpCompiler -> "A[(42 - 2)]", + GoCompiler -> "this.A[(42 - 2)]", + JavaCompiler -> "a().get((42 - 2))", + JavaScriptCompiler -> "this.a[(42 - 2)]", + LuaCompiler -> "self.a[(43 - 2)]", + PHPCompiler -> "$this->a()[(42 - 2)]", + PythonCompiler -> "self.a[(42 - 2)]", + RubyCompiler -> "a[(42 - 2)]" + )) + + full("a.first", ArrayType(CalcIntType), CalcIntType, Map[LanguageCompilerStatic, String]( + CppCompiler -> "a()->front()", + CSharpCompiler -> "A[0]", + GoCompiler -> "this.A[0]", + JavaCompiler -> "a().get(0)", + JavaScriptCompiler -> "this.a[0]", + LuaCompiler -> "self.a[1]", + PHPCompiler -> "$this->a()[0]", + PythonCompiler -> "self.a[0]", + RubyCompiler -> "a.first" + )) + + full("a.last", ArrayType(CalcIntType), CalcIntType, Map[LanguageCompilerStatic, String]( + CppCompiler -> "a()->back()", + CSharpCompiler -> "A[A.Count - 1]", + GoCompiler -> "this.A[len(this.A)-1]", + JavaCompiler -> "a().get(a().size() - 1)", + JavaScriptCompiler -> "this.a[this.a.length - 1]", + LuaCompiler -> "self.a[#self.a]", + PHPCompiler -> "$this->a()[count($this->a()) - 1]", + PythonCompiler -> "self.a[-1]", + RubyCompiler -> "a.last" + )) + + full("a.size", ArrayType(CalcIntType), CalcIntType, Map[LanguageCompilerStatic, String]( + CppCompiler -> "a()->size()", + CSharpCompiler -> "A.Count", + GoCompiler -> "len(this.A)", + JavaCompiler -> "a().size()", + JavaScriptCompiler -> "this.a.length", + LuaCompiler -> "#self.a", + PHPCompiler -> "count($this->a())", + PerlCompiler -> "scalar($self->a())", + PythonCompiler -> "len(self.a)", + RubyCompiler -> "a.length" + )) + + // Strings + full("\"str\"", CalcIntType, CalcStrType, Map[LanguageCompilerStatic, String]( + CppCompiler -> "std::string(\"str\")", + CSharpCompiler -> "\"str\"", + GoCompiler -> "\"str\"", + JavaCompiler -> "\"str\"", + JavaScriptCompiler -> "\"str\"", + LuaCompiler -> "\"str\"", + PerlCompiler -> "\"str\"", + PHPCompiler -> "\"str\"", + PythonCompiler -> "u\"str\"", + RubyCompiler -> "\"str\"" + )) + + full("\"str\\nnext\"", CalcIntType, CalcStrType, Map[LanguageCompilerStatic, String]( + CppCompiler -> "std::string(\"str\\nnext\")", + CSharpCompiler -> "\"str\\nnext\"", + GoCompiler -> "\"str\\nnext\"", + JavaCompiler -> "\"str\\nnext\"", + JavaScriptCompiler -> "\"str\\nnext\"", + LuaCompiler -> "\"str\\nnext\"", + PerlCompiler -> "\"str\\nnext\"", + PHPCompiler -> "\"str\\nnext\"", + PythonCompiler -> "u\"str\\nnext\"", + RubyCompiler -> "\"str\\nnext\"" + )) + + full("\"str\\u000anext\"", CalcIntType, CalcStrType, Map[LanguageCompilerStatic, String]( + CppCompiler -> "std::string(\"str\\nnext\")", + CSharpCompiler -> "\"str\\nnext\"", + GoCompiler -> "\"str\\u000anext\"", + JavaCompiler -> "\"str\\nnext\"", + JavaScriptCompiler -> "\"str\\nnext\"", + LuaCompiler -> "\"str\\nnext\"", + PerlCompiler -> "\"str\\nnext\"", + PHPCompiler -> "\"str\\nnext\"", + PythonCompiler -> "u\"str\\nnext\"", + RubyCompiler -> "\"str\\nnext\"" + )) + + full("\"str\\0next\"", CalcIntType, CalcStrType, Map[LanguageCompilerStatic, String]( + CppCompiler -> "std::string(\"str\\000next\", 8)", + CSharpCompiler -> "\"str\\0next\"", + GoCompiler -> "\"str\\000next\"", + JavaCompiler -> "\"str\\000next\"", + JavaScriptCompiler -> "\"str\\000next\"", + LuaCompiler -> "\"str\\000next\"", + PerlCompiler -> "\"str\\000next\"", + PHPCompiler -> "\"str\\000next\"", + PythonCompiler -> "u\"str\\000next\"", + RubyCompiler -> "\"str\\000next\"" + )) + + everybodyExcept("\"str1\" + \"str2\"", "\"str1\" + \"str2\"", Map[LanguageCompilerStatic, String]( + CppCompiler -> "std::string(\"str1\") + std::string(\"str2\")", + LuaCompiler -> "\"str1\" .. \"str2\"", + PerlCompiler -> "\"str1\" . \"str2\"", + PHPCompiler -> "\"str1\" . \"str2\"", + PythonCompiler -> "u\"str1\" + u\"str2\"" + ), CalcStrType) + + everybodyExcept("\"str1\" == \"str2\"", "\"str1\" == \"str2\"", Map[LanguageCompilerStatic, String]( + CppCompiler -> "std::string(\"str1\") == (std::string(\"str2\"))", + JavaCompiler -> "\"str1\".equals(\"str2\")", + LuaCompiler -> "\"str1\" == \"str2\"", + PerlCompiler -> "\"str1\" eq \"str2\"", + PythonCompiler -> "u\"str1\" == u\"str2\"" + ), CalcBooleanType) + + everybodyExcept("\"str1\" != \"str2\"", "\"str1\" != \"str2\"", Map[LanguageCompilerStatic, String]( + CppCompiler -> "std::string(\"str1\") != std::string(\"str2\")", + JavaCompiler -> "!(\"str1\").equals(\"str2\")", + LuaCompiler -> "\"str1\" ~= \"str2\"", + PerlCompiler -> "\"str1\" ne \"str2\"", + PythonCompiler -> "u\"str1\" != u\"str2\"" + ), CalcBooleanType) + + everybodyExcept("\"str1\" < \"str2\"", "\"str1\" < \"str2\"", Map[LanguageCompilerStatic, String]( + CppCompiler -> "(std::string(\"str1\").compare(std::string(\"str2\")) < 0)", + CSharpCompiler -> "(\"str1\".CompareTo(\"str2\") < 0)", + JavaCompiler -> "(\"str1\".compareTo(\"str2\") < 0)", + LuaCompiler -> "\"str1\" < \"str2\"", + PerlCompiler -> "\"str1\" lt \"str2\"", + PythonCompiler -> "u\"str1\" < u\"str2\"" + ), CalcBooleanType) + + full("\"str\".length", CalcIntType, CalcIntType, Map[LanguageCompilerStatic, String]( + CppCompiler -> "std::string(\"str\").length()", + CSharpCompiler -> "\"str\".Length", + GoCompiler -> "utf8.RuneCountInString(\"str\")", + JavaCompiler -> "\"str\".length()", + JavaScriptCompiler -> "\"str\".length", + LuaCompiler -> "string.len(\"str\")", + PerlCompiler -> "length(\"str\")", + PHPCompiler -> "strlen(\"str\")", + PythonCompiler -> "len(u\"str\")", + RubyCompiler -> "\"str\".size" + )) + + full("\"str\".reverse", CalcIntType, CalcStrType, Map[LanguageCompilerStatic, String]( + CppCompiler -> "kaitai::kstream::reverse(std::string(\"str\"))", + CSharpCompiler -> "new string(Array.Reverse(\"str\".ToCharArray()))", + GoCompiler -> "kaitai.StringReverse(\"str\")", + JavaCompiler -> "new StringBuilder(\"str\").reverse().toString()", + JavaScriptCompiler -> "Array.from(\"str\").reverse().join('')", + LuaCompiler -> "string.reverse(\"str\")", + PerlCompiler -> "scalar(reverse(\"str\"))", + PHPCompiler -> "strrev(\"str\")", + PythonCompiler -> "u\"str\"[::-1]", + RubyCompiler -> "\"str\".reverse" + )) + + full("\"12345\".to_i", CalcIntType, CalcIntType, Map[LanguageCompilerStatic, String]( + CppCompiler -> "std::stoi(std::string(\"12345\"))", + CSharpCompiler -> "Convert.ToInt64(\"12345\", 10)", + GoCompiler -> "func()(int){i, err := strconv.Atoi(\"12345\"); if (err != nil) { panic(err) }; return i}()", + JavaCompiler -> "Long.parseLong(\"12345\", 10)", + JavaScriptCompiler -> "Number.parseInt(\"12345\", 10)", + LuaCompiler -> "tonumber(\"12345\")", + PerlCompiler -> "\"12345\"", + PHPCompiler -> "intval(\"12345\", 10)", + PythonCompiler -> "int(u\"12345\")", + RubyCompiler -> "\"12345\".to_i" + )) + + full("\"1234fe\".to_i(16)", CalcIntType, CalcIntType, Map[LanguageCompilerStatic, String]( + CppCompiler -> "std::stoi(std::string(\"1234fe\"), 0, 16)", + CSharpCompiler -> "Convert.ToInt64(\"1234fe\", 16)", + GoCompiler -> "func()(int64){i, err := strconv.ParseInt(\"1234fe\", 16, 64); if (err != nil) { panic(err) }; return i}()", + JavaCompiler -> "Long.parseLong(\"1234fe\", 16)", + JavaScriptCompiler -> "Number.parseInt(\"1234fe\", 16)", + LuaCompiler -> "tonumber(\"1234fe\", 16)", + PerlCompiler -> "hex(\"1234fe\")", + PHPCompiler -> "intval(\"1234fe\", 16)", + PythonCompiler -> "int(u\"1234fe\", 16)", + RubyCompiler -> "\"1234fe\".to_i(16)" + )) + + // casts + full("other.as.bar", FooBarProvider, CalcStrType, Map[LanguageCompilerStatic, String]( + CppCompiler -> "static_cast(other())->bar()", + CSharpCompiler -> "((Block) (Other)).Bar", + GoCompiler -> "this.Other.(Block).Bar", + JavaCompiler -> "((Block) (other())).bar()", + JavaScriptCompiler -> "this.other.bar", + LuaCompiler -> "self.other.bar", + PerlCompiler -> "$self->other()->bar()", + PHPCompiler -> "$this->other()->bar()", + PythonCompiler -> "self.other.bar", + RubyCompiler -> "other.bar" + )) + + full("other.as.baz", FooBarProvider, CalcIntType, Map[LanguageCompilerStatic, String]( + CppCompiler -> "static_cast(other())->baz()", + CSharpCompiler -> "((Block.Innerblock) (Other)).Baz", + GoCompiler -> "this.Other.(Block.Innerblock).Baz", + JavaCompiler -> "((Block.Innerblock) (other())).baz()", + JavaScriptCompiler -> "this.other.baz", + LuaCompiler -> "self.other.baz", + PerlCompiler -> "$self->other()->baz()", + PHPCompiler -> "$this->other()->baz()", + PythonCompiler -> "self.other.baz", + RubyCompiler -> "other.baz" + )) + + // primitive pure types + full("(1 + 2).as", CalcIntType, IntMultiType(true, Width2, None), Map[LanguageCompilerStatic, String]( + CppCompiler -> "static_cast((1 + 2))", + CSharpCompiler -> "((short) ((1 + 2)))", + GoCompiler -> "int16((1 + 2))", + JavaCompiler -> "((short) ((1 + 2)))", + JavaScriptCompiler -> "(1 + 2)", + LuaCompiler -> "(1 + 2)", + PerlCompiler -> "(1 + 2)", + PHPCompiler -> "(1 + 2)", + PythonCompiler -> "(1 + 2)", + RubyCompiler -> "(1 + 2)" + )) + + // empty array casting + full("[].as", CalcIntType, CalcBytesType, Map[LanguageCompilerStatic, String]( + CppCompiler -> "std::string(\"\", 0)", + CSharpCompiler -> "new byte[] { }", + GoCompiler -> "\"\"", + JavaCompiler -> "new byte[] { }", + JavaScriptCompiler -> "[]", + LuaCompiler -> "\"\"", + PerlCompiler -> "pack('C*', ())", + PHPCompiler -> "\"\"", + PythonCompiler -> "b\"\"", + RubyCompiler -> "[].pack('C*')" + )) + + full("[].as", CalcIntType, ArrayType(Int1Type(false)), Map[LanguageCompilerStatic, String]( + CppCompiler -> "std::string(\"\")", + CSharpCompiler -> "new List { }", + GoCompiler -> "[]uint8{}", + JavaCompiler -> "new ArrayList(Arrays.asList())", + JavaScriptCompiler -> "[]", + LuaCompiler -> "{}", + PerlCompiler -> "()", + PHPCompiler -> "[]", + PythonCompiler -> "[]", + RubyCompiler -> "[]" + )) + + full("[].as", CalcIntType, ArrayType(FloatMultiType(Width8, None)), Map[LanguageCompilerStatic, String]( + CppCompiler -> "std::string(\"\", 0)", + CSharpCompiler -> "new List { }", + GoCompiler -> "[]float64{}", + JavaCompiler -> "new ArrayList(Arrays.asList())", + JavaScriptCompiler -> "[]", + LuaCompiler -> "{}", + PerlCompiler -> "()", + PHPCompiler -> "[]", + PythonCompiler -> "[]", + RubyCompiler -> "[]" + )) + + // type enforcement: casting to non-literal byte array + full("[0 + 1, 5].as", CalcIntType, CalcBytesType, Map[LanguageCompilerStatic, String]( + CppCompiler -> "???", + CSharpCompiler -> "new byte[] { (0 + 1), 5 }", + GoCompiler -> "string([]byte{(0 + 1), 5})", + JavaCompiler -> "new byte[] { (0 + 1), 5 }", + JavaScriptCompiler -> "new Uint8Array([(0 + 1), 5])", + LuaCompiler -> "???", + PerlCompiler -> "pack('C*', ((0 + 1), 5))", + PHPCompiler -> "pack('C*', (0 + 1), 5)", + PythonCompiler -> "struct.pack('2b', (0 + 1), 5)", + RubyCompiler -> "[(0 + 1), 5].pack('C*')" + )) + + // type enforcement: casting to array of integers + full("[0, 1, 2].as", CalcIntType, ArrayType(Int1Type(false)), Map[LanguageCompilerStatic, String]( + CSharpCompiler -> "new List { 0, 1, 2 }", + GoCompiler -> "[]uint8{0, 1, 2}", + JavaCompiler -> "new ArrayList(Arrays.asList(0, 1, 2))", + JavaScriptCompiler -> "[0, 1, 2]", + LuaCompiler -> "{0, 1, 2}", + PerlCompiler -> "(0, 1, 2)", + PHPCompiler -> "[0, 1, 2]", + PythonCompiler -> "[0, 1, 2]", + RubyCompiler -> "[0, 1, 2]" + )) + + // sizeof of primitive types + everybody("sizeof", "1", CalcIntType) + everybody("sizeof", "1", CalcIntType) + everybody("sizeof", "1", CalcIntType) + everybody("sizeof", "2", CalcIntType) + everybody("sizeof", "1", CalcIntType) + everybody("sizeof", "2", CalcIntType) + everybody("sizeof", "4", CalcIntType) + everybody("sizeof", "8", CalcIntType) + + // sizeof of fixed user type + everybody("sizeof", "7", CalcIntType) + + // bitsizeof of primitive types + everybody("bitsizeof", "1", CalcIntType) + everybody("bitsizeof", "7", CalcIntType) + everybody("bitsizeof", "8", CalcIntType) + everybody("bitsizeof", "9", CalcIntType) + everybody("bitsizeof", "8", CalcIntType) + everybody("bitsizeof", "16", CalcIntType) + everybody("bitsizeof", "32", CalcIntType) + everybody("bitsizeof", "64", CalcIntType) + + // sizeof of fixed user type + everybody("bitsizeof", "56", CalcIntType) + + def runTest(src: String, tp: TypeProvider, expType: DataType, expOut: ResultMap) { var eo: Option[Ast.expr] = None test(s"_expr:$src") { eo = Some(Expressions.parse(src)) } - val langs = Map[LanguageCompilerStatic, BaseTranslator]( - CppCompiler -> new CppTranslator(tp), - CSharpCompiler -> new CSharpTranslator(tp), - JavaCompiler -> new JavaTranslator(tp, RuntimeConfig()), + val goOutput = new StringLanguageOutputWriter(" ") + + val langs = Map[LanguageCompilerStatic, AbstractTranslator with TypeDetector]( + CppCompiler -> new CppTranslator(tp, new ImportList(), RuntimeConfig()), + CSharpCompiler -> new CSharpTranslator(tp, new ImportList()), + GoCompiler -> new GoTranslator(goOutput, tp, new ImportList()), + JavaCompiler -> new JavaTranslator(tp, new ImportList()), JavaScriptCompiler -> new JavaScriptTranslator(tp), - PerlCompiler -> new PerlTranslator(tp), + LuaCompiler -> new LuaTranslator(tp, new ImportList()), + PerlCompiler -> new PerlTranslator(tp, new ImportList()), PHPCompiler -> new PHPTranslator(tp, RuntimeConfig()), - PythonCompiler -> new PythonTranslator(tp), + PythonCompiler -> new PythonTranslator(tp, new ImportList()), RubyCompiler -> new RubyTranslator(tp) ) langs.foreach { case (langObj, tr) => val langName = LanguageCompilerStatic.CLASS_TO_NAME(langObj) - test(s"$langName:$src") { + test(s"$langName:$src", Tag(langName), Tag(src)) { eo match { case Some(e) => - val tr: BaseTranslator = langs(langObj) + val tr: AbstractTranslator with TypeDetector = langs(langObj) expOut.get(langObj) match { case Some(expResult) => tr.detectType(e) should be(expType) - tr.translate(e) should be(expResult) + val actResult1 = tr.translate(e) + val actResult2 = langObj match { + case GoCompiler => goOutput.result + actResult1 + case _ => actResult1 + } + actResult2 should be(expResult) case None => fail("no expected result") } @@ -438,11 +647,25 @@ class TranslatorSpec extends FunSuite with TableDrivenPropertyChecks { abstract class FakeTypeProvider extends TypeProvider { val nowClass = ClassSpec.opaquePlaceholder(List("top_class")) - override def resolveEnum(enumName: String) = + override def resolveEnum(inType: Ast.typeId, enumName: String) = throw new NotImplementedError - override def resolveType(typeName: String): DataType = - throw new NotImplementedError + override def resolveType(typeName: Ast.typeId): DataType = { + if (typeName == Ast.typeId(false, List("block"), false)) { + val name = List("top_class", "block") + val r = CalcUserType(name, None, Seq()) + val cs = ClassSpec.opaquePlaceholder(name) + cs.seqSize = FixedSized(56) + r.classSpec = Some(cs) + return r + } else { + throw new NotImplementedError + } + } + + override def isLazy(attrName: String): Boolean = false + + override def isLazy(inClass: ClassSpec, attrName: String): Boolean = false } case class Always(t: DataType) extends FakeTypeProvider { @@ -450,30 +673,57 @@ class TranslatorSpec extends FunSuite with TableDrivenPropertyChecks { override def determineType(inClass: ClassSpec, name: String): DataType = t } + /** + * Emulates the following system of types: + * + * 
+    *   meta:
+    *     id: top_class
+    *   types:
+    *     block:
+    *       seq:
+    *         - id: bar
+    *           type: str
+    *         - id: inner
+    *           type: innerblock
+    *       types:
+    *         innerblock:
+    *           instances:
+    *             baz:
+    *               value: 123
+    *
+ */ case object FooBarProvider extends FakeTypeProvider { override def determineType(name: String): DataType = { name match { - case "foo" => userType("block") + case "foo" => userType(List("top_class", "block")) } } override def determineType(inClass: ClassSpec, name: String): DataType = { - (inClass.name, name) match { - case (List("block"), "bar") => CalcStrType - case (List("block"), "inner") => userType("innerblock") - case (List("innerblock"), "baz") => CalcIntType + (inClass.name.last, name) match { + case ("block", "bar") => CalcStrType + case ("block", "inner") => userType(List("top_class", "block", "innerblock")) + case ("innerblock", "baz") => CalcIntType } } - override def resolveType(typeName: String): DataType = { - typeName match { - case "top_class" | "block" | "innerblock" => userType(typeName) + override def resolveType(typeName: Ast.typeId): DataType = { + typeName.names match { + case Seq("top_class") => + userType(List("top_class")) + case Seq("block") | + Seq("top_class", "block") => + userType(List("top_class", "block")) + case Seq("innerblock") | + Seq("block", "innerblock") | + Seq("top_class", "block", "innerblock") => + userType(List("top_class", "block", "innerblock")) } } } - def userType(name: String) = { - val lname = List(name) + def userType(lname: List[String]) = { val cs = ClassSpec.opaquePlaceholder(lname) val ut = UserTypeInstream(lname, None) ut.classSpec = Some(cs) @@ -482,19 +732,17 @@ class TranslatorSpec extends FunSuite with TableDrivenPropertyChecks { lazy val ALL_LANGS = LanguageCompilerStatic.NAME_TO_CLASS.values - def full(src: String, srcType: DataType, expType: DataType, expOut: ResultMap): TestSpec = - (src, Always(srcType), expType, expOut) + def full(src: String, srcType: DataType, expType: DataType, expOut: ResultMap) = + runTest(src, Always(srcType), expType, expOut) - def full(src: String, tp: TypeProvider, expType: DataType, expOut: ResultMap): TestSpec = - (src, tp, expType, expOut) + def full(src: String, tp: TypeProvider, expType: DataType, expOut: ResultMap) = + runTest(src, tp, expType, expOut) - def everybody(src: String, expOut: String, expType: DataType = CalcIntType): TestSpec = { - (src, Always(CalcIntType), expType, ALL_LANGS.map((langObj) => langObj -> expOut).toMap) - } + def everybody(src: String, expOut: String, expType: DataType = CalcIntType) = + runTest(src, Always(CalcIntType), expType, ALL_LANGS.map((langObj) => langObj -> expOut).toMap) - def everybodyExcept(src: String, commonExpOut: String, rm: ResultMap, expType: DataType = CalcIntType): TestSpec = { - (src, Always(CalcIntType), expType, ALL_LANGS.map((langObj) => + def everybodyExcept(src: String, commonExpOut: String, rm: ResultMap, expType: DataType = CalcIntType) = + runTest(src, Always(CalcIntType), expType, ALL_LANGS.map((langObj) => langObj -> rm.getOrElse(langObj, commonExpOut) ).toMap) - } } diff --git a/jvm/src/test/scala/io/kaitai/struct/translators/TypeDetector$Test.scala b/jvm/src/test/scala/io/kaitai/struct/translators/TypeDetector$Test.scala new file mode 100644 index 000000000..1e81a236e --- /dev/null +++ b/jvm/src/test/scala/io/kaitai/struct/translators/TypeDetector$Test.scala @@ -0,0 +1,16 @@ +package io.kaitai.struct.translators + +import io.kaitai.struct.datatype.DataType._ +import org.scalatest.FunSpec +import org.scalatest.Matchers._ + +class TypeDetector$Test extends FunSpec { + describe("TypeDetector") { + it("combines ints properly") { + val ut1 = CalcUserType(List("foo"), None) + val ut2 = CalcUserType(List("bar"), None) + + TypeDetector.combineTypes(ut1, ut2) should be(CalcKaitaiStructType) + } + } +} diff --git a/lib_bintray.sh b/lib_bintray.sh new file mode 100644 index 000000000..6ccc6faee --- /dev/null +++ b/lib_bintray.sh @@ -0,0 +1,84 @@ +# Shell library to handle uploads & publishing of artifacts to Bintray + +# All functions get their settings from global variables: +# +# * Authentication: +# * BINTRAY_USER +# * BINTRAY_API_KEY - to be passed as secret env variable# +# * Package ID / upload coordinates: +# * BINTRAY_ACCOUNT +# * BINTRAY_REPO +# * BINTRAY_PACKAGE +# * BINTRAY_VERSION +# * Debian-specific settings: +# * BINTRAY_DEB_DISTRIBUTION +# * BINTRAY_DEB_ARCH +# * BINTRAY_DEB_COMPONENT +# * Debug options: +# * BINTRAY_CURL_ARGS - set to `-vv` for verbose output + +## +# Creates version for a package at Bintray +bintray_create_version() +{ + echo "bintray_create_version(repo=${BINTRAY_REPO}, package=${BINTRAY_PACKAGE}, version=${BINTRAY_VERSION})" + + curl $BINTRAY_CURL_ARGS -f \ + "-u$BINTRAY_USER:$BINTRAY_API_KEY" \ + -H "Content-Type: application/json" \ + -X POST "https://api.bintray.com/packages/${BINTRAY_ACCOUNT}/${BINTRAY_REPO}/${BINTRAY_PACKAGE}/versions" \ + --data "{ \"name\": \"$BINTRAY_VERSION\", \"release_notes\": \"auto\", \"released\": \"\" }" +# --data "{ \"name\": \"$version\", \"release_notes\": \"auto\", \"release_url\": \"$BASE_DESC/$RPM_NAME\", \"released\": \"\" }" +} + +## +# Uploads generic file to Bintray. +# +# Input: +# $1 = filename to upload +bintray_upload_generic() +{ + local filename="$1" + + echo "bintray_upload_generic(repo=${BINTRAY_REPO}, package=${BINTRAY_PACKAGE}, version=${BINTRAY_VERSION}, filename=${filename})" + + curl $BINTRAY_CURL_ARGS -f \ + -T "$filename" \ + "-u$BINTRAY_USER:$BINTRAY_API_KEY" \ + -H "X-Bintray-Package: $BINTRAY_PACKAGE" \ + -H "X-Bintray-Version: $BINTRAY_VERSION" \ + https://api.bintray.com/content/$BINTRAY_ACCOUNT/$BINTRAY_REPO/ +} + +## +# Uploads deb package to Bintray. +# +# Input: +# $1 = filename to upload +bintray_upload_deb() +{ + local filename="$1" + + echo "bintray_upload_deb(repo=${BINTRAY_REPO}, package=${BINTRAY_PACKAGE}, version=${BINTRAY_VERSION}, filename=${filename})" + + curl $BINTRAY_CURL_ARGS -f \ + -T "$filename" \ + "-u$BINTRAY_USER:$BINTRAY_API_KEY" \ + -H "X-Bintray-Package: $BINTRAY_PACKAGE" \ + -H "X-Bintray-Version: $BINTRAY_VERSION" \ + -H "X-Bintray-Debian-Distribution: $BINTRAY_DEB_DISTRIBUTION" \ + -H "X-Bintray-Debian-Architecture: $BINTRAY_DEB_ARCH" \ + -H "X-Bintray-Debian-Component: $BINTRAY_DEB_COMPONENT" \ + https://api.bintray.com/content/$BINTRAY_ACCOUNT/$BINTRAY_REPO/ +} + +bintray_publish_version() +{ + echo "bintray_publish_version(repo=${BINTRAY_REPO}, package=${BINTRAY_PACKAGE}, version=${BINTRAY_VERSION})" + + curl $BINTRAY_CURL_ARGS -f \ + "-u$BINTRAY_USER:$BINTRAY_API_KEY" \ + -H "Content-Type: application/json" \ + -X POST "https://api.bintray.com/content/$BINTRAY_ACCOUNT/$BINTRAY_REPO/$BINTRAY_PACKAGE/$BINTRAY_VERSION/publish" \ + --data "{ \"discard\": \"false\" }" +} diff --git a/project/build.properties b/project/build.properties index d638b4f34..cabf73b45 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1 @@ -sbt.version = 0.13.8 \ No newline at end of file +sbt.version = 1.2.7 diff --git a/project/plugins.sbt b/project/plugins.sbt index 2b65a24e6..52104a18b 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -1,5 +1,4 @@ logLevel := Level.Warn -addSbtPlugin("com.typesafe.sbt" % "sbt-native-packager" % "1.2.0-M8") -addSbtPlugin("com.eed3si9n" % "sbt-buildinfo" % "0.6.1") -addSbtPlugin("org.scala-js" % "sbt-scalajs" % "0.6.14") +addSbtPlugin("com.typesafe.sbt" % "sbt-native-packager" % "1.3.12") +addSbtPlugin("org.scala-js" % "sbt-scalajs" % "0.6.28") diff --git a/publish_deb_to_bintray.sh b/publish_deb_to_bintray.sh new file mode 100755 index 000000000..28a557ab6 --- /dev/null +++ b/publish_deb_to_bintray.sh @@ -0,0 +1,21 @@ +#!/bin/sh -ef + +. ./lib_bintray.sh + +# Config +BINTRAY_USER=greycat +BINTRAY_ACCOUNT=kaitai-io +BINTRAY_REPO=debian_unstable +BINTRAY_PACKAGE=kaitai-struct-compiler +BINTRAY_VERSION="$KAITAI_STRUCT_VERSION" +# BINTRAY_API_KEY comes from encrypted variables from web UI + +BINTRAY_DEB_DISTRIBUTION=jessie +BINTRAY_DEB_ARCH=all +BINTRAY_DEB_COMPONENT=main + +#BINTRAY_CURL_ARGS=-v + +bintray_create_version +bintray_upload_deb "jvm/target/kaitai-struct-compiler_${KAITAI_STRUCT_VERSION}_all.deb" +bintray_publish_version diff --git a/publish_js_to_npm.sh b/publish_js_to_npm.sh new file mode 100755 index 000000000..428a7eb39 --- /dev/null +++ b/publish_js_to_npm.sh @@ -0,0 +1,10 @@ +#!/bin/sh + +# js/npm is a directory where we deploy contents of NPM package that +# will be eventually published. When calling this script, this +# directory should already exist and be populated. + +cd js/npm +echo "//registry.npmjs.org/:_authToken=$NPM_API_KEY" >$HOME/.npmrc +npm publish --tag next +rm ~/.npmrc diff --git a/publish_zip_to_bintray.sh b/publish_zip_to_bintray.sh new file mode 100755 index 000000000..a21705393 --- /dev/null +++ b/publish_zip_to_bintray.sh @@ -0,0 +1,17 @@ +#!/bin/sh -ef + +. ./lib_bintray.sh + +# Config +BINTRAY_USER=greycat +BINTRAY_ACCOUNT=kaitai-io +BINTRAY_REPO=universal_unstable +BINTRAY_PACKAGE=kaitai-struct-compiler +BINTRAY_VERSION="$KAITAI_STRUCT_VERSION" +# BINTRAY_API_KEY comes from encrypted variables from web UI + +#BINTRAY_CURL_ARGS=-v + +bintray_create_version +bintray_upload_generic "jvm/target/universal/kaitai-struct-compiler-${KAITAI_STRUCT_VERSION}.zip" +bintray_publish_version diff --git a/shared/src/main/scala/io/kaitai/struct/ClassCompiler.scala b/shared/src/main/scala/io/kaitai/struct/ClassCompiler.scala index 552c8a69f..c29cbe393 100644 --- a/shared/src/main/scala/io/kaitai/struct/ClassCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/ClassCompiler.scala @@ -1,19 +1,19 @@ package io.kaitai.struct import io.kaitai.struct.CompileLog.FileSuccess -import io.kaitai.struct.datatype.DataType import io.kaitai.struct.datatype.DataType._ -import io.kaitai.struct.format._ -import io.kaitai.struct.languages.components.{LanguageCompiler, LanguageCompilerStatic} - -import scala.collection.mutable.ListBuffer +import io.kaitai.struct.datatype._ +import io.kaitai.struct.exprlang.Ast +import io.kaitai.struct.format.{AttrSpec, _} +import io.kaitai.struct.languages.components.{ExtraAttrs, LanguageCompiler, LanguageCompilerStatic} class ClassCompiler( + classSpecs: ClassSpecs, val topClass: ClassSpec, config: RuntimeConfig, langObj: LanguageCompilerStatic ) extends AbstractCompiler { - val provider = new ClassTypeProvider(topClass) + val provider = new ClassTypeProvider(classSpecs, topClass) val topClassName = topClass.name val lang: LanguageCompiler = langObj.getCompiler(provider, config) @@ -36,16 +36,18 @@ class ClassCompiler( ) } + /** + * Generates code for one full class using a given [[ClassSpec]]. + * @param curClass current class to generate code for + */ def compileClass(curClass: ClassSpec): Unit = { provider.nowClass = curClass - if (!curClass.doc.isEmpty) - lang.classDoc(curClass.name, curClass.doc) + if (!lang.innerDocstrings) + compileClassDoc(curClass) lang.classHeader(curClass.name) - - val extraAttrs = ListBuffer[AttrSpec]() - extraAttrs += AttrSpec(List(), RootIdentifier, UserTypeInstream(topClassName, None)) - extraAttrs += AttrSpec(List(), ParentIdentifier, curClass.parentType) + if (lang.innerDocstrings) + compileClassDoc(curClass) // Forward declarations for recursive types curClass.types.foreach { case (typeName, _) => lang.classForwardDeclaration(List(typeName)) } @@ -53,33 +55,22 @@ class ClassCompiler( if (lang.innerEnums) compileEnums(curClass) - if (lang.debug) + if (lang.config.readStoresPos) lang.debugClassSequence(curClass.seq) - lang.classConstructorHeader(curClass.name, curClass.parentTypeName, topClassName) - curClass.instances.foreach { case (instName, _) => lang.instanceClear(instName) } - compileSeqRead(curClass.seq, extraAttrs) - lang.classConstructorFooter + // Constructor + compileConstructor(curClass) - if (config.readWrite) { - lang.funcWriteHeader(curClass) - compileSeqWrite(curClass.seq, extraAttrs) - lang.funcWriteFooter(curClass) + // Read method(s) + compileEagerRead(curClass.seq, curClass.meta.endian) - lang.funcCheckHeader(curClass) - compileSeqCheck(curClass.seq) - lang.funcCheckFooter(curClass) + if (config.readWrite) { + compileWrite(curClass.seq, curClass.meta.endian) + compileCheck(curClass.seq) } - lang.classDestructorHeader(curClass.name, curClass.parentTypeName, topClassName) - curClass.seq.foreach((attr) => lang.attrDestructor(attr, attr.id)) - curClass.instances.foreach { case (id, instSpec) => - instSpec match { - case pis: ParseInstanceSpec => lang.attrDestructor(pis, id) - case _: ValueInstanceSpec => // ignore for now - } - } - lang.classDestructorFooter + // Destructor + compileDestructor(curClass) // Recursive types if (lang.innerClasses) { @@ -88,16 +79,19 @@ class ClassCompiler( provider.nowClass = curClass } - curClass.instances.foreach { case (instName, instSpec) => compileInstance(curClass.name, instName, instSpec, extraAttrs) } + compileInstances(curClass) // Attributes declarations and readers - compileAttrDeclarations(curClass.seq ++ extraAttrs) - - (curClass.seq ++ extraAttrs).foreach { (attr) => - if (!attr.doc.isEmpty) - lang.attributeDoc(attr.id, attr.doc) - lang.attributeReader(attr.id, attr.dataTypeComposite, attr.cond) - } + val allAttrs: List[MemberSpec] = + curClass.seq ++ + curClass.params ++ + List( + AttrSpec(List(), RootIdentifier, CalcUserType(topClassName, None)), + AttrSpec(List(), ParentIdentifier, curClass.parentType) + ) ++ + ExtraAttrs.forClassSpec(curClass, lang) + compileAttrDeclarations(allAttrs) + compileAttrReaders(allAttrs) lang.classFooter(curClass.name) @@ -108,27 +102,225 @@ class ClassCompiler( compileEnums(curClass) } - def compileAttrDeclarations(attrs: List[AttrSpec]): Unit = - attrs.foreach((attr) => lang.attributeDeclaration(attr.id, attr.dataTypeComposite, attr.cond)) + /** + * Compiles constructor for a given class. Generally, it should: + * + * * store passed parameters, io/root/parent/endianness if needed + * * initialize everything + * * invoke _read() method, if applicable + * + * @param curClass current class to generate code for + */ + def compileConstructor(curClass: ClassSpec) = { + lang.classConstructorHeader( + curClass.name, + curClass.parentType, + topClassName, + curClass.meta.endian.contains(InheritedEndian), + curClass.params + ) + compileInit(curClass) + curClass.instances.foreach { case (instName, _) => lang.instanceClear(instName) } + if (lang.config.autoRead) + lang.runRead() + lang.classConstructorFooter + } + + /** + * Compile initialization of class members for a given type. Typically + * this is only required for languages which both: + * + * * don't perform auto-initialization of object with some default + * values (like 0s) on object creation, + * * require these members to be initialized because any other + * procedures with object (e.g. destruction) will require that + * + * Currently, this is only applicable to C++ without smart pointers, + * as destructors we'll generate will rely on pointers being set to + * null. + * @param curClass current type to generate code for + */ + def compileInit(curClass: ClassSpec) = { + curClass.seq.foreach((attr) => compileAttrInit(attr)) + curClass.instances.foreach { case (_, instSpec) => + instSpec match { + case pis: ParseInstanceSpec => compileAttrInit(pis) + case _: ValueInstanceSpec => // ignore for now + } + } + } + + def compileAttrInit(originalAttr: AttrLikeSpec): Unit = { + val extraAttrs = ExtraAttrs.forAttr(originalAttr, lang) + val allAttrs = List(originalAttr) ++ extraAttrs + allAttrs.foreach((attr) => lang.attrInit(attr)) + } + + /** + * Compiles destructor for a given type. It should clean up everything + * (i.e. every applicable allocated seq / instance attribute variables, and + * any extra attribute variables, if they were used). + * @param curClass current type to generate code for + */ + def compileDestructor(curClass: ClassSpec) = { + lang.classDestructorHeader(curClass.name, curClass.parentType, topClassName) + curClass.seq.foreach((attr) => lang.attrDestructor(attr, attr.id)) + curClass.instances.foreach { case (id, instSpec) => + instSpec match { + case pis: ParseInstanceSpec => lang.attrDestructor(pis, id) + case _: ValueInstanceSpec => // ignore for now + } + } + lang.classDestructorFooter + } + + /** + * Iterates over a given list of attributes and generates attribute + * declarations for each of them. + * @param attrs attribute list to traverse + */ + def compileAttrDeclarations(attrs: List[MemberSpec]): Unit = { + attrs.foreach { (attr) => + val isNullable = if (lang.switchBytesOnlyAsRaw) { + attr.isNullableSwitchRaw + } else { + attr.isNullable + } + lang.attributeDeclaration(attr.id, attr.dataTypeComposite, isNullable) + } + } + + /** + * Iterates over a given list of attributes and generates attribute + * readers (AKA getters) for each of them. + * @param attrs attribute list to traverse + */ + def compileAttrReaders(attrs: List[MemberSpec]): Unit = { + attrs.foreach { (attr) => + // FIXME: Python should have some form of attribute docs too + if (!attr.doc.isEmpty && !lang.innerDocstrings) + lang.attributeDoc(attr.id, attr.doc) + val isNullable = if (lang.switchBytesOnlyAsRaw) { + attr.isNullableSwitchRaw + } else { + attr.isNullable + } + lang.attributeReader(attr.id, attr.dataTypeComposite, isNullable) + } + } + + /** + * Compiles everything related to "eager reading" for a given list of + * sequence attributes and endianness. Depending on endianness: + * + * * For types known to have fixed endianness, we do just "_read" method. + * * For types with ambiguous endianness, we'll do `_read` + "_read_le" + + * "_read_be" methods. If endianness needs to be calculated, we'll perform + * that calculation in "_read". If it's inherited, then we'll just make + * decision based on that inherited setting. + * + * @param seq list of sequence attributes + * @param endian endianness setting + */ + def compileEagerRead(seq: List[AttrSpec], endian: Option[Endianness]): Unit = { + endian match { + case None | Some(_: FixedEndian) => + compileSeqReadProc(seq, None) + case Some(ce: CalcEndian) => + lang.readHeader(None, false) + compileCalcEndian(ce) + lang.runReadCalc() + lang.readFooter() + + compileSeqReadProc(seq, Some(LittleEndian)) + compileSeqReadProc(seq, Some(BigEndian)) + case Some(InheritedEndian) => + lang.readHeader(None, false) + lang.runReadCalc() + lang.readFooter() + + compileSeqReadProc(seq, Some(LittleEndian)) + compileSeqReadProc(seq, Some(BigEndian)) + } + } + + def compileWrite(seq: List[AttrSpec], endian: Option[Endianness]): Unit = { + endian match { + case None | Some(_: FixedEndian) => + compileSeqWriteProc(seq, None) + case Some(CalcEndian(_, _)) | Some(InheritedEndian) => + lang.writeHeader(None) + lang.runWriteCalc() + lang.writeFooter() + + compileSeqWriteProc(seq, Some(LittleEndian)) + compileSeqWriteProc(seq, Some(BigEndian)) + } + } + + def compileCheck(seq: List[AttrSpec]): Unit = { + lang.checkHeader() + compileSeqCheck(seq) + lang.checkFooter() + } + + val IS_LE_ID = SpecialIdentifier("_is_le") + + /** + * Compiles endianness calculation procedure and stores result in a special + * attribute [[IS_LE_ID]]. Typically occurs as part of "_read" method. + * @param ce calculated endianness specification + */ + def compileCalcEndian(ce: CalcEndian): Unit = { + def renderProc(result: FixedEndian): Unit = { + val v = Ast.expr.Bool(result == LittleEndian) + lang.instanceCalculate(IS_LE_ID, CalcBooleanType, v) + } + + lang.switchCases[FixedEndian](IS_LE_ID, ce.on, ce.cases, renderProc, renderProc) + } - def compileSeqRead(seq: List[AttrSpec], extraAttrs: ListBuffer[AttrSpec]) = { + /** + * Compiles seq reading method (complete with header and footer). + * @param seq sequence of attributes + * @param defEndian default endianness + */ + def compileSeqReadProc(seq: List[AttrSpec], defEndian: Option[FixedEndian]) = { + lang.readHeader(defEndian, seq.isEmpty) + compileSeqRead(seq, defEndian) + lang.readFooter() + } + + def compileSeqWriteProc(seq: List[AttrSpec], defEndian: Option[FixedEndian]) = { + lang.writeHeader(defEndian) + compileSeqWrite(seq, defEndian) + lang.writeFooter() + } + + /** + * Compiles seq reading method body (only reading statements). + * @param seq sequence of attributes + * @param defEndian default endianness + */ + def compileSeqRead(seq: List[AttrSpec], defEndian: Option[FixedEndian]) = { var wasUnaligned = false seq.foreach { (attr) => val nowUnaligned = isUnalignedBits(attr.dataType) if (wasUnaligned && !nowUnaligned) lang.alignToByte(lang.normalIO) - lang.attrParse(attr, attr.id, extraAttrs) + lang.attrParse(attr, attr.id, defEndian) + compileValidate(attr) wasUnaligned = nowUnaligned } } - def compileSeqWrite(seq: List[AttrSpec], extraAttrs: ListBuffer[AttrSpec]) = { + def compileSeqWrite(seq: List[AttrSpec], defEndian: Option[FixedEndian]) = { var wasUnaligned = false seq.foreach { (attr) => val nowUnaligned = isUnalignedBits(attr.dataType) if (wasUnaligned && !nowUnaligned) lang.alignToByte(lang.normalIO) - lang.attrWrite(attr, attr.id, extraAttrs) + lang.attrWrite(attr, attr.id, defEndian) wasUnaligned = nowUnaligned } } @@ -139,27 +331,45 @@ class ClassCompiler( } } + /** + * Compiles validation procedure for one attribute after it was parsed. + * @param attr attribute to validate + */ + def compileValidate(attr: AttrSpec): Unit = + attr.valid.foreach(valid => lang.attrValidate(attr.id, attr, valid)) + + /** + * Compiles all enums specifications for a given type. + * @param curClass current type to generate code for + */ def compileEnums(curClass: ClassSpec): Unit = curClass.enums.foreach { case(_, enumColl) => compileEnum(curClass, enumColl) } + /** + * Compile subclasses for a given class. + * @param curClass current type to generate code for + */ def compileSubclasses(curClass: ClassSpec): Unit = curClass.types.foreach { case (_, intClass) => compileClass(intClass) } - def compileInstance(className: List[String], instName: InstanceIdentifier, instSpec: InstanceSpec, extraAttrs: ListBuffer[AttrSpec]): Unit = { + def compileInstances(curClass: ClassSpec) = { + curClass.instances.foreach { case (instName, instSpec) => + compileInstance(curClass.name, instName, instSpec, curClass.meta.endian) + } + } + + def compileInstance(className: List[String], instName: InstanceIdentifier, instSpec: InstanceSpec, endian: Option[Endianness]): Unit = { // Determine datatype val dataType = instSpec.dataTypeComposite - // Declare caching variable - val condSpec = instSpec match { - case vis: ValueInstanceSpec => ConditionalSpec(vis.ifExpr, NoRepeat) - case pis: ParseInstanceSpec => pis.cond - } - lang.instanceDeclaration(instName, dataType, condSpec) + compileInstanceDeclaration(instName, instSpec) - if (!instSpec.doc.isEmpty) - lang.attributeDoc(instName, instSpec.doc) - lang.instanceHeader(className, instName, dataType) - lang.instanceCheckCacheAndReturn(instName) + if (!lang.innerDocstrings) + compileInstanceDoc(instName, instSpec) + lang.instanceHeader(className, instName, dataType, instSpec.isNullable) + if (lang.innerDocstrings) + compileInstanceDoc(instName, instSpec) + lang.instanceCheckCacheAndReturn(instName, dataType) instSpec match { case vi: ValueInstanceSpec => @@ -167,17 +377,19 @@ class ClassCompiler( lang.instanceCalculate(instName, dataType, vi.value) lang.attrParseIfFooter(vi.ifExpr) case i: ParseInstanceSpec => - lang.attrParse(i, instName, extraAttrs) + lang.attrParse(i, instName, endian) } lang.instanceSetCalculated(instName) - lang.instanceReturn(instName) + lang.instanceReturn(instName, dataType) lang.instanceFooter } - def compileEnum(curClass: ClassSpec, enumColl: EnumSpec): Unit = { + def compileInstanceDeclaration(instName: InstanceIdentifier, instSpec: InstanceSpec): Unit = + lang.instanceDeclaration(instName, instSpec.dataTypeComposite, instSpec.isNullable) + + def compileEnum(curClass: ClassSpec, enumColl: EnumSpec): Unit = lang.enumDeclaration(curClass.name, enumColl.name.last, enumColl.sortedSeq) - } def isUnalignedBits(dt: DataType): Boolean = dt match { @@ -185,4 +397,14 @@ class ClassCompiler( case et: EnumType => isUnalignedBits(et.basedOn) case _ => false } + + def compileClassDoc(curClass: ClassSpec) = { + if (!curClass.doc.isEmpty) + lang.classDoc(curClass.name, curClass.doc) + } + + def compileInstanceDoc(instName: Identifier, instSpec: InstanceSpec) = { + if (!instSpec.doc.isEmpty) + lang.attributeDoc(instName, instSpec.doc) + } } diff --git a/shared/src/main/scala/io/kaitai/struct/ClassTypeProvider.scala b/shared/src/main/scala/io/kaitai/struct/ClassTypeProvider.scala index 6f443e920..bd9317fa7 100644 --- a/shared/src/main/scala/io/kaitai/struct/ClassTypeProvider.scala +++ b/shared/src/main/scala/io/kaitai/struct/ClassTypeProvider.scala @@ -2,11 +2,12 @@ package io.kaitai.struct import io.kaitai.struct.datatype.DataType import io.kaitai.struct.datatype.DataType._ +import io.kaitai.struct.exprlang.Ast import io.kaitai.struct.format._ import io.kaitai.struct.precompile.{EnumNotFoundError, FieldNotFoundError, TypeNotFoundError, TypeUndecidedError} import io.kaitai.struct.translators.TypeProvider -class ClassTypeProvider(topClass: ClassSpec) extends TypeProvider { +class ClassTypeProvider(classSpecs: ClassSpecs, var topClass: ClassSpec) extends TypeProvider { var nowClass = topClass var _currentIteratorType: Option[DataType] = None @@ -24,28 +25,34 @@ class ClassTypeProvider(topClass: ClassSpec) extends TypeProvider { makeUserType(topClass) case Identifier.PARENT => if (inClass.parentClass == UnknownClassSpec) - throw new RuntimeException(s"Unable to derive _parent type in ${inClass.name.mkString("::")}") + throw new RuntimeException(s"Unable to derive ${Identifier.PARENT} type in ${inClass.name.mkString("::")}") makeUserType(inClass.parentClass) case Identifier.IO => KaitaiStreamType case Identifier.ITERATOR => currentIteratorType - case Identifier.ITERATOR_I => - CalcIntType - case "_on" => + case Identifier.SWITCH_ON => currentSwitchType + case Identifier.INDEX => + CalcIntType + case Identifier.SIZEOF => + CalcIntType case _ => inClass.seq.foreach { el => if (el.id == NamedIdentifier(attrName)) return el.dataTypeComposite } + inClass.params.foreach { el => + if (el.id == NamedIdentifier(attrName)) + return el.dataType + } inClass.instances.get(InstanceIdentifier(attrName)) match { case Some(i: ValueInstanceSpec) => - i.dataType match { + val dt = i.dataType match { case Some(t) => t case None => throw new TypeUndecidedError(attrName) } - return i.dataType.get + return dt case Some(i: ParseInstanceSpec) => return i.dataTypeComposite case None => // do nothing } @@ -58,13 +65,14 @@ class ClassTypeProvider(topClass: ClassSpec) extends TypeProvider { case GenericStructClassSpec => KaitaiStructType case cs: ClassSpec => - val ut = UserTypeInstream(cs.name, None) + val ut = CalcUserType(cs.name, None) ut.classSpec = Some(cs) ut } } - override def resolveEnum(enumName: String): EnumSpec = resolveEnum(nowClass, enumName) + override def resolveEnum(inType: Ast.typeId, enumName: String): EnumSpec = + resolveEnum(resolveClassSpec(inType), enumName) def resolveEnum(inClass: ClassSpec, enumName: String): EnumSpec = { inClass.enums.get(enumName) match { @@ -80,21 +88,66 @@ class ClassTypeProvider(topClass: ClassSpec) extends TypeProvider { } } - override def resolveType(typeName: String): DataType = resolveType(nowClass, typeName) + override def resolveType(typeName: Ast.typeId): DataType = + makeUserType(resolveClassSpec(typeName)) + + def resolveClassSpec(typeName: Ast.typeId): ClassSpec = + resolveClassSpec( + if (typeName.absolute) topClass else nowClass, + typeName.names + ) + + def resolveClassSpec(inClass: ClassSpec, typeName: Seq[String]): ClassSpec = { + if (typeName.isEmpty) + return inClass + + val headTypeName :: restTypesNames = typeName.toList + val nextClass = resolveClassSpec(inClass, headTypeName) + if (restTypesNames.isEmpty) { + nextClass + } else { + resolveClassSpec(nextClass, restTypesNames) + } + } + + def resolveClassSpec(inClass: ClassSpec, typeName: String): ClassSpec = { + if (inClass.name.last == typeName) + return inClass - def resolveType(inClass: ClassSpec, typeName: String): DataType = { inClass.types.get(typeName) match { case Some(spec) => - val ut = UserTypeInstream(spec.name, None) - ut.classSpec = Some(spec) - ut + spec case None => // let's try upper levels of hierarchy inClass.upClass match { - case Some(upClass) => resolveType(upClass, typeName) + case Some(upClass) => resolveClassSpec(upClass, typeName) case None => - throw new TypeNotFoundError(typeName, nowClass) + classSpecs.get(typeName) match { + case Some(spec) => spec + case None => + throw new TypeNotFoundError(typeName, nowClass) + } } } } + + override def isLazy(attrName: String): Boolean = isLazy(nowClass, attrName) + + def isLazy(inClass: ClassSpec, attrName: String): Boolean = { + inClass.seq.foreach { el => + if (el.id == NamedIdentifier(attrName)) + return false + } + inClass.params.foreach { el => + if (el.id == NamedIdentifier(attrName)) + return false + } + inClass.instances.get(InstanceIdentifier(attrName)) match { + case Some(i) => + return true + case None => + // do nothing + } + throw new FieldNotFoundError(attrName, inClass) + } } diff --git a/shared/src/main/scala/io/kaitai/struct/CompileLog.scala b/shared/src/main/scala/io/kaitai/struct/CompileLog.scala index b5d8d03f3..134494b1e 100644 --- a/shared/src/main/scala/io/kaitai/struct/CompileLog.scala +++ b/shared/src/main/scala/io/kaitai/struct/CompileLog.scala @@ -1,10 +1,18 @@ package io.kaitai.struct +/** + * Namespace for all the objects related to compilation results. + */ object CompileLog { - sealed trait InputEntry extends Jsonable + trait CanHasErrors { + def hasErrors: Boolean + } + + sealed trait InputEntry extends Jsonable with CanHasErrors case class InputFailure(errors: List[CompileError]) extends InputEntry { override def toJson: String = JSON.mapToJson(Map("errors" -> errors)) + override def hasErrors = true } case class InputSuccess( @@ -15,13 +23,17 @@ object CompileLog { "firstSpecName" -> firstSpecName, "output" -> output )) + + override def hasErrors: Boolean = + output.values.map(_.values.map(_.hasErrors).max).max } /** Compilation result of a single [[io.kaitai.struct.format.ClassSpec]] into a single target language. */ - sealed trait SpecEntry extends Jsonable + sealed trait SpecEntry extends Jsonable with CanHasErrors case class SpecFailure(errors: List[CompileError]) extends SpecEntry { override def toJson: String = JSON.mapToJson(Map("errors" -> errors)) + override def hasErrors: Boolean = true } case class SpecSuccess( @@ -32,6 +44,7 @@ object CompileLog { "topLevelName" -> topLevelName, "files" -> files )) + override def hasErrors: Boolean = false } case class FileSuccess( diff --git a/shared/src/main/scala/io/kaitai/struct/ConstructClassCompiler.scala b/shared/src/main/scala/io/kaitai/struct/ConstructClassCompiler.scala new file mode 100644 index 000000000..a4b6a3c53 --- /dev/null +++ b/shared/src/main/scala/io/kaitai/struct/ConstructClassCompiler.scala @@ -0,0 +1,223 @@ +package io.kaitai.struct + +import io.kaitai.struct.datatype.DataType._ +import io.kaitai.struct.datatype._ +import io.kaitai.struct.exprlang.Ast +import io.kaitai.struct.format._ +import io.kaitai.struct.languages.components.{LanguageCompiler, LanguageCompilerStatic} +import io.kaitai.struct.translators.ConstructTranslator + +class ConstructClassCompiler(classSpecs: ClassSpecs, topClass: ClassSpec) extends AbstractCompiler { + val out = new StringLanguageOutputWriter(indent) + val importList = new ImportList + + val provider = new ClassTypeProvider(classSpecs, topClass) + val translator = new ConstructTranslator(provider, importList) + + override def compile: CompileLog.SpecSuccess = { + out.puts("from construct import *") + out.puts("from construct.lib import *") + out.puts + + compileClass(topClass) + + out.puts(s"_schema = ${type2class(topClass)}") + + CompileLog.SpecSuccess( + "", + List(CompileLog.FileSuccess( + outFileName(topClass.nameAsStr), + out.result + )) + ) + } + + def compileClass(cs: ClassSpec): Unit = { + cs.types.foreach { case (_, typeSpec) => compileClass(typeSpec) } + + cs.enums.foreach { case (_, enumSpec) => compileEnum(enumSpec) } + + out.puts(s"${type2class(cs)} = Struct(") + out.inc + + provider.nowClass = cs + + cs.seq.foreach((seqAttr) => compileAttr(seqAttr)) + cs.instances.foreach { case (id, instSpec) => + instSpec match { + case vis: ValueInstanceSpec => + compileValueInstance(id, vis) + case pis: ParseInstanceSpec => + compileParseInstance(pis) + } + } + + out.dec + out.puts(")") + out.puts + } + + def compileAttr(attr: AttrLikeSpec): Unit = { + out.puts(s"'${idToStr(attr.id)}' / ${compileAttrBody(attr)},") + } + + def compileValueInstance(id: Identifier, vis: ValueInstanceSpec): Unit = { + val typeStr = s"Computed(lambda this: ${translator.translate(vis.value)})" + val typeStr2 = wrapWithIf(typeStr, vis.ifExpr) + out.puts(s"'${idToStr(id)}' / $typeStr2,") + } + + def compileParseInstance(attr: ParseInstanceSpec): Unit = { + attr.pos match { + case None => + compileAttr(attr) + case Some(pos) => + out.puts(s"'${idToStr(attr.id)}' / " + + s"Pointer(${translator.translate(pos)}, ${compileAttrBody(attr)}),") + } + } + + def compileAttrBody(attr: AttrLikeSpec): String = { + val typeStr1 = typeToStr(attr.dataType) + val typeStr2 = wrapWithRepeat(typeStr1, attr.cond.repeat, attr.dataType) + wrapWithIf(typeStr2, attr.cond.ifExpr) + } + + def wrapWithRepeat(typeStr: String, repeat: RepeatSpec, dataType: DataType) = repeat match { + case RepeatExpr(expr) => + s"Array(${translator.translate(expr)}, $typeStr)" + case RepeatUntil(expr) => + provider._currentIteratorType = Some(dataType) + s"RepeatUntil(lambda obj_, list_, this: ${translator.translate(expr)}, $typeStr)" + case RepeatEos => + s"GreedyRange($typeStr)" + case NoRepeat => + typeStr + } + + def wrapWithIf(typeStr: String, ifExpr: Option[Ast.expr]) = ifExpr match { + case Some(expr) => s"If(${translator.translate(expr)}, $typeStr)" + case None => typeStr + } + + def compileEnum(enumSpec: EnumSpec): Unit = { + out.puts(s"def ${enumToStr(enumSpec)}(subcon):") + out.inc + out.puts("return Enum(subcon,") + out.inc + enumSpec.sortedSeq.foreach { case (number, valueSpec) => + out.puts(s"${valueSpec.name}=$number,") + } + out.dec + out.puts(")") + out.dec + out.puts + } + + def idToStr(id: Identifier): String = { + id match { + case SpecialIdentifier(name) => name + case NamedIdentifier(name) => name + case NumberedIdentifier(idx) => s"_${NumberedIdentifier.TEMPLATE}$idx" + case InstanceIdentifier(name) => name + } + } + + def type2class(cs: ClassSpec) = cs.name.mkString("__") + + def enumToStr(enumSpec: EnumSpec) = enumSpec.name.mkString("__") + + def typeToStr(dataType: DataType): String = dataType match { + case fbt: FixedBytesType => + s"Const(${translator.doByteArrayLiteral(fbt.contents)})" + case Int1Type(signed) => + s"Int8${signToStr(signed)}b" + case IntMultiType(signed, width, endianOpt) => + s"Int${width.width * 8}${signToStr(signed)}${fixedEndianToStr(endianOpt.get)}" + case FloatMultiType(width, endianOpt) => + s"Float${width.width * 8}${fixedEndianToStr(endianOpt.get)}" + case BytesEosType(terminator, include, padRight, process) => + "GreedyBytes" + case blt: BytesLimitType => + attrBytesLimitType(blt) + case btt: BytesTerminatedType => + attrBytesTerminatedType(btt, "GreedyBytes") + case StrFromBytesType(bytes, encoding) => + bytes match { + case BytesEosType(terminator, include, padRight, process) => + s"GreedyString(encoding='$encoding')" + case blt: BytesLimitType => + attrBytesLimitType(blt, s"GreedyString(encoding='$encoding')") + case btt: BytesTerminatedType => + attrBytesTerminatedType(btt, s"GreedyString(encoding='$encoding')") + } + case ut: UserTypeInstream => + s"LazyBound(lambda: ${type2class(ut.classSpec.get)})" + case utb: UserTypeFromBytes => + utb.bytes match { + //case BytesEosType(terminator, include, padRight, process) => + case BytesLimitType(size, terminator, include, padRight, process) => + s"FixedSized(${translator.translate(size)}, LazyBound(lambda: ${type2class(utb.classSpec.get)}))" + //case BytesTerminatedType(terminator, include, consume, eosError, process) => + case _ => "???" + } + case et: EnumType => + s"${enumToStr(et.enumSpec.get)}(${typeToStr(et.basedOn)})" + case st: SwitchType => + attrSwitchType(st) + case _ => "???" + } + + def attrBytesLimitType(blt: BytesLimitType, subcon: String = "GreedyBytes"): String = { + val subcon2 = blt.terminator match { + case None => subcon + case Some(term) => + val termStr = "\\x%02X".format(term & 0xff) + s"NullTerminated($subcon, term=b'$termStr', include=${translator.doBoolLiteral(blt.include)})" + } + val subcon3 = blt.padRight match { + case None => subcon2 + case Some(padRight) => + val padStr = "\\x%02X".format(padRight & 0xff) + s"NullStripped($subcon2, pad=b'$padStr')" + } + s"FixedSized(${translator.translate(blt.size)}, $subcon3)" + } + + def attrBytesTerminatedType(btt: BytesTerminatedType, subcon: String): String = { + val termStr = "\\x%02X".format(btt.terminator & 0xff) + s"NullTerminated($subcon, " + + s"term=b'$termStr', " + + s"include=${translator.doBoolLiteral(btt.include)}, " + + s"consume=${translator.doBoolLiteral(btt.consume)})" + } + + def attrSwitchType(st: SwitchType): String = { + val cases = st.cases.filter { case (caseExpr, _) => + caseExpr != SwitchType.ELSE_CONST + }.map { case (caseExpr, caseType) => + s"${translator.translate(caseExpr)}: ${typeToStr(caseType)}, " + } + + val defaultSuffix = st.cases.get(SwitchType.ELSE_CONST).map((t) => + s", default=${typeToStr(t)}" + ).getOrElse("") + + s"Switch(${translator.translate(st.on)}, {${cases.mkString}}$defaultSuffix)" + } + + def signToStr(signed: Boolean) = if (signed) "s" else "u" + + def fixedEndianToStr(e: FixedEndian) = e match { + case LittleEndian => "l" + case BigEndian => "b" + } + + def indent: String = "\t" + def outFileName(topClassName: String): String = s"$topClassName.py" +} + +object ConstructClassCompiler extends LanguageCompilerStatic { + // FIXME: Unused, should be probably separated from LanguageCompilerStatic + override def getCompiler(tp: ClassTypeProvider, config: RuntimeConfig): LanguageCompiler = ??? +} diff --git a/shared/src/main/scala/io/kaitai/struct/DocClassCompiler.scala b/shared/src/main/scala/io/kaitai/struct/DocClassCompiler.scala new file mode 100644 index 000000000..339a6c20b --- /dev/null +++ b/shared/src/main/scala/io/kaitai/struct/DocClassCompiler.scala @@ -0,0 +1,84 @@ +package io.kaitai.struct + +import io.kaitai.struct.format._ +import io.kaitai.struct.precompile.CalculateSeqSizes +import io.kaitai.struct.translators.RubyTranslator + +abstract class DocClassCompiler(classSpecs: ClassSpecs, topClass: ClassSpec) extends AbstractCompiler { + val provider = new ClassTypeProvider(classSpecs, topClass) + val translator = new RubyTranslator(provider) + + // TODO: move it into SingleOutputFile equivalent + val out = new StringLanguageOutputWriter(indent) + def outFileName(topClass: ClassSpec): String + def indent: String + // END move to SingleOutputFile + + def nowClass: ClassSpec = provider.nowClass + def nowClassName = provider.nowClass.name + + override def compile: CompileLog.SpecSuccess = { + fileHeader(topClass) + compileClass(topClass) + fileFooter(topClass) + + CompileLog.SpecSuccess( + "", + List(CompileLog.FileSuccess( + outFileName(topClass), + out.result + )) + ) + } + + def compileClass(curClass: ClassSpec): Unit = { + provider.nowClass = curClass + + classHeader(curClass) + + // Sequence + compileSeqRead(curClass) + + // Instances + curClass.instances.foreach { case (_, instSpec) => + instSpec match { + case pis: ParseInstanceSpec => + compileParseInstance(curClass, pis) + case vis: ValueInstanceSpec => + compileValueInstance(vis) + } + } + + // Enums + curClass.enums.foreach { case(enumName, enumColl) => compileEnum(enumName, enumColl) } + + // Recursive types + curClass.types.foreach { case (_, intClass) => compileClass(intClass) } + + classFooter(curClass) + } + + def compileSeqRead(curClass: ClassSpec): Unit = { + seqHeader(curClass) + + CalculateSeqSizes.forEachSeqAttr(curClass, (attr, seqPos, sizeElement, sizeContainer) => { + compileSeqAttr(curClass, attr, seqPos, sizeElement, sizeContainer) + }) + + seqFooter(curClass) + } + + def fileHeader(topClass: ClassSpec): Unit + def fileFooter(topClass: ClassSpec): Unit + + def classHeader(classSpec: ClassSpec): Unit + def classFooter(classSpec: ClassSpec): Unit + + def seqHeader(classSpec: ClassSpec): Unit + def seqFooter(classSpec: ClassSpec): Unit + + def compileSeqAttr(classSpec: ClassSpec, attr: AttrSpec, seqPos: Option[Int], sizeElement: Sized, sizeContainer: Sized): Unit + def compileParseInstance(classSpec: ClassSpec, inst: ParseInstanceSpec): Unit + def compileValueInstance(vis: ValueInstanceSpec): Unit + def compileEnum(enumName: String, enumColl: EnumSpec): Unit +} diff --git a/shared/src/main/scala/io/kaitai/struct/GoClassCompiler.scala b/shared/src/main/scala/io/kaitai/struct/GoClassCompiler.scala new file mode 100644 index 000000000..9c5af1597 --- /dev/null +++ b/shared/src/main/scala/io/kaitai/struct/GoClassCompiler.scala @@ -0,0 +1,95 @@ +package io.kaitai.struct + +import io.kaitai.struct.datatype.DataType.{CalcIntType, KaitaiStreamType, UserTypeInstream} +import io.kaitai.struct.datatype.{BigEndian, CalcEndian, Endianness, FixedEndian, InheritedEndian, LittleEndian} +import io.kaitai.struct.exprlang.Ast +import io.kaitai.struct.format._ +import io.kaitai.struct.languages.GoCompiler +import io.kaitai.struct.languages.components.ExtraAttrs + +class GoClassCompiler( + classSpecs: ClassSpecs, + override val topClass: ClassSpec, + config: RuntimeConfig +) extends ClassCompiler(classSpecs, topClass, config, GoCompiler) { + + override def compileClass(curClass: ClassSpec): Unit = { + provider.nowClass = curClass + + val extraAttrs = List( + AttrSpec(List(), IoIdentifier, KaitaiStreamType), + AttrSpec(List(), RootIdentifier, UserTypeInstream(topClassName, None)), + AttrSpec(List(), ParentIdentifier, curClass.parentType) + ) ++ ExtraAttrs.forClassSpec(curClass, lang) + + if (!curClass.doc.isEmpty) + lang.classDoc(curClass.name, curClass.doc) + + // Enums declaration defines types, so they need to go first + compileEnums(curClass) + + // Basic struct declaration + lang.classHeader(curClass.name) + compileAttrDeclarations(curClass.seq ++ extraAttrs) + curClass.instances.foreach { case (instName, instSpec) => + compileInstanceDeclaration(instName, instSpec) + } + lang.classFooter(curClass.name) + + // Constructor = Read() function + compileReadFunction(curClass) + + compileInstances(curClass) + + compileAttrReaders(curClass.seq ++ extraAttrs) + + // Recursive types + compileSubclasses(curClass) + } + + def compileReadFunction(curClass: ClassSpec) = { + lang.classConstructorHeader( + curClass.name, + curClass.parentType, + topClassName, + curClass.meta.endian.contains(InheritedEndian), + curClass.params + ) + compileEagerRead(curClass.seq, curClass.meta.endian) + lang.classConstructorFooter + } + + override def compileInstance(className: List[String], instName: InstanceIdentifier, instSpec: InstanceSpec, endian: Option[Endianness]): Unit = { + // Determine datatype + val dataType = instSpec.dataTypeComposite + + if (!instSpec.doc.isEmpty) + lang.attributeDoc(instName, instSpec.doc) + lang.instanceHeader(className, instName, dataType, instSpec.isNullable) + lang.instanceCheckCacheAndReturn(instName, dataType) + + instSpec match { + case vi: ValueInstanceSpec => + lang.attrParseIfHeader(instName, vi.ifExpr) + lang.instanceCalculate(instName, dataType, vi.value) + lang.attrParseIfFooter(vi.ifExpr) + case i: ParseInstanceSpec => + lang.attrParse(i, instName, endian) + } + + lang.instanceSetCalculated(instName) + lang.instanceReturn(instName, dataType) + lang.instanceFooter + } + + override def compileCalcEndian(ce: CalcEndian): Unit = { + def renderProc(result: FixedEndian): Unit = { + val v = result match { + case LittleEndian => Ast.expr.IntNum(1) + case BigEndian => Ast.expr.IntNum(0) + } + lang.instanceCalculate(IS_LE_ID, CalcIntType, v) + } + lang.switchCases[FixedEndian](IS_LE_ID, ce.on, ce.cases, renderProc, renderProc) + } +} diff --git a/shared/src/main/scala/io/kaitai/struct/GraphvizClassCompiler.scala b/shared/src/main/scala/io/kaitai/struct/GraphvizClassCompiler.scala index b0939f882..32b87ff83 100644 --- a/shared/src/main/scala/io/kaitai/struct/GraphvizClassCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/GraphvizClassCompiler.scala @@ -1,21 +1,21 @@ package io.kaitai.struct -import io.kaitai.struct.exprlang.Ast -import io.kaitai.struct.exprlang.Ast.expr import io.kaitai.struct.datatype.DataType import io.kaitai.struct.datatype.DataType._ +import io.kaitai.struct.exprlang.Ast import io.kaitai.struct.format._ import io.kaitai.struct.languages.components.{LanguageCompiler, LanguageCompilerStatic} -import io.kaitai.struct.translators.{BaseTranslator, RubyTranslator, TypeProvider} +import io.kaitai.struct.precompile.CalculateSeqSizes +import io.kaitai.struct.translators.RubyTranslator import scala.collection.mutable.ListBuffer -class GraphvizClassCompiler(topClass: ClassSpec) extends AbstractCompiler { +class GraphvizClassCompiler(classSpecs: ClassSpecs, topClass: ClassSpec) extends AbstractCompiler { import GraphvizClassCompiler._ val out = new StringLanguageOutputWriter(indent) - val provider = new ClassTypeProvider(topClass) + val provider = new ClassTypeProvider(classSpecs, topClass) val translator = new RubyTranslator(provider) val links = ListBuffer[(String, String, String)]() val extraClusterLines = new StringLanguageOutputWriter(indent) @@ -59,7 +59,7 @@ class GraphvizClassCompiler(topClass: ClassSpec) extends AbstractCompiler { out.puts // Sequence - compileSeq(className, curClass) + compileSeqRead(className, curClass) curClass.instances.foreach { case (instName, instSpec) => instSpec match { @@ -84,34 +84,19 @@ class GraphvizClassCompiler(topClass: ClassSpec) extends AbstractCompiler { out.puts("}") } - def compileSeq(className: List[String], curClass: ClassSpec): Unit = { + def compileSeqRead(className: List[String], curClass: ClassSpec): Unit = { tableStart(className, "seq") - var seqPos: Option[Int] = Some(0) - curClass.seq.foreach { (attr) => + + CalculateSeqSizes.forEachSeqAttr(curClass, (attr, seqPos, _, _) => { attr.id match { case NamedIdentifier(name) => tableRow(className, seqPosToStr(seqPos), attr, name) - - val size = dataTypeBitsSize(attr.dataType) - seqPos = (seqPos, size) match { - case (Some(pos), Some(siz)) => Some(pos + siz) - case _ => None - } + case NumberedIdentifier(n) => + tableRow(className, seqPosToStr(seqPos), attr, s"_${NumberedIdentifier.TEMPLATE}$n") } - } - tableEnd - } + }) - def seqPosToStr(seqPos: Option[Int]): Option[String] = { - seqPos.map { (pos) => - val posByte = pos / 8 - val posBit = pos % 8 - if (posBit == 0) { - s"$posByte" - } else { - s"$posByte:$posBit" - } - } + tableEnd } def compileParseInstance(className: List[String], id: InstanceIdentifier, inst: ParseInstanceSpec): Unit = { @@ -254,63 +239,66 @@ class GraphvizClassCompiler(topClass: ClassSpec) extends AbstractCompiler { */ def dataTypeSizeAsString(dataType: DataType, attrName: String): String = { dataType match { - case _: Int1Type => "1" - case IntMultiType(_, width, _) => width.width.toString - case FloatMultiType(width, _) => width.width.toString - case FixedBytesType(contents, _) => contents.length.toString case _: BytesEosType => END_OF_STREAM case blt: BytesLimitType => expressionSize(blt.size, attrName) - case _: BytesTerminatedType => UNKNOWN case StrFromBytesType(basedOn, _) => dataTypeSizeAsString(basedOn, attrName) case utb: UserTypeFromBytes => dataTypeSizeAsString(utb.bytes, attrName) - case UserTypeInstream(_, _) => UNKNOWN case EnumType(_, basedOn) => dataTypeSizeAsString(basedOn, attrName) - case _: SwitchType => UNKNOWN - case BitsType1 => "1b" - case BitsType(width) => s"${width}b" + case _ => + CalculateSeqSizes.dataTypeBitsSize(dataType) match { + case FixedSized(n) => + if (n % 8 == 0) { + s"${n / 8}" + } else { + s"${n}b" + } + case DynamicSized => UNKNOWN + case NotCalculatedSized | StartedCalculationSized => + throw new RuntimeException("Should never happen: problems with CalculateSeqSizes") + } } } - def expressionSize(ex: expr, attrName: String): String = { + def expressionSize(ex: Ast.expr, attrName: String): String = { expression(ex, getGraphvizNode(nowClassName, nowClass, attrName) + s":${attrName}_size", STYLE_EDGE_SIZE) } - def expressionPos(ex: expr, attrName: String): String = { + def expressionPos(ex: Ast.expr, attrName: String): String = { expression(ex, getGraphvizNode(nowClassName, nowClass, attrName) + s":${attrName}_pos", STYLE_EDGE_POS) } - def expressionType(ex: expr, attrName: String): String = { + def expressionType(ex: Ast.expr, attrName: String): String = { expression(ex, getGraphvizNode(nowClassName, nowClass, attrName) + s":${attrName}_type", STYLE_EDGE_VALUE) } - def expression(e: expr, portName: String, style: String): String = { + def expression(e: Ast.expr, portName: String, style: String): String = { affectedVars(e).foreach((v) => links += ((v, portName, style)) ) htmlEscape(translator.translate(e)) } - def affectedVars(e: expr): List[String] = { + def affectedVars(e: Ast.expr): List[String] = { e match { - case expr.BoolOp(op, values) => + case Ast.expr.BoolOp(op, values) => values.flatMap(affectedVars).toList - case expr.BinOp(left, op, right) => + case Ast.expr.BinOp(left, op, right) => affectedVars(left) ++ affectedVars(right) - case expr.UnaryOp(op, operand) => + case Ast.expr.UnaryOp(op, operand) => affectedVars(operand) - case expr.IfExp(condition, ifTrue, ifFalse) => + case Ast.expr.IfExp(condition, ifTrue, ifFalse) => affectedVars(condition) ++ affectedVars(ifTrue) ++ affectedVars(ifFalse) // case expr.Dict(keys, values) => - case expr.Compare(left, ops, right) => + case Ast.expr.Compare(left, ops, right) => affectedVars(left) ++ affectedVars(right) // case expr.Call(func, args) => - case expr.IntNum(_) | expr.FloatNum(_) | expr.Str(_) | expr.Bool(_) => + case Ast.expr.IntNum(_) | Ast.expr.FloatNum(_) | Ast.expr.Str(_) | Ast.expr.Bool(_) => List() - case expr.EnumByLabel(enumName, label) => + case _: Ast.expr.EnumByLabel => List() - case expr.EnumById(enumName, id) => + case Ast.expr.EnumById(_, id, _) => affectedVars(id) - case expr.Attribute(value, attr) => + case Ast.expr.Attribute(value, attr) => val targetClass = translator.detectType(value) targetClass match { case t: UserType => @@ -327,17 +315,20 @@ class GraphvizClassCompiler(topClass: ClassSpec) extends AbstractCompiler { case _ => affectedVars(value) } - case expr.Subscript(value, idx) => + case Ast.expr.Subscript(value, idx) => affectedVars(value) ++ affectedVars(idx) case SwitchType.ELSE_CONST => // "_" is a special const for List() - case expr.Name(Ast.identifier("_io")) => - // "_io" is a special const too - List() - case expr.Name(id) => - List(resolveLocalNode(id.name)) - case expr.List(elts) => + case Ast.expr.Name(id) => + if (id.name.charAt(0) == '_') { + // other special consts like "_io", "_index", etc + List() + } else { + // this must be local name, resolve it + List(resolveLocalNode(id.name)) + } + case Ast.expr.List(elts) => elts.flatMap(affectedVars).toList } } @@ -367,14 +358,29 @@ class GraphvizClassCompiler(topClass: ClassSpec) extends AbstractCompiler { s"${getGraphvizNode(className, cs, s)}:${s}_type" def getGraphvizNode(className: List[String], cs: ClassSpec, s: String): String = { - cs.seq.foreach((attr) => - attr.id match { + cs.seq.foreach { (attr) => + val name = attr.id match { case NamedIdentifier(attrName) => - if (attrName == s) { - return s"${type2class(className)}__seq" - } + attrName + case NumberedIdentifier(n) => + s"_${NumberedIdentifier.TEMPLATE}$n" } - ) + if (name == s) { + return s"${type2class(className)}__seq" + } + } + + cs.params.foreach { (attr) => + val name = attr.id match { + case NamedIdentifier(attrName) => + attrName + case NumberedIdentifier(n) => + s"_${NumberedIdentifier.TEMPLATE}$n" + } + if (name == s) { + return s"${type2class(className)}__params" + } + } cs.instances.get(InstanceIdentifier(s)).foreach((inst) => return s"${type2class(className)}__inst__$s" @@ -397,46 +403,9 @@ object GraphvizClassCompiler extends LanguageCompilerStatic { def type2class(name: List[String]) = name.last def type2display(name: List[String]) = name.map(Utils.upperCamelCase).mkString("::") - /** - * Determines how many bits occupies given data type. - * - * @param dataType data type to analyze - * @return number of bits or None, if it's impossible to determine a priori - */ - def dataTypeBitsSize(dataType: DataType): Option[Int] = { - dataType match { - case BitsType1 => Some(1) - case BitsType(width) => Some(width) - case EnumType(_, basedOn) => dataTypeBitsSize(basedOn) - case _ => dataTypeByteSize(dataType).map((byteSize) => byteSize * 8) - } - } - - /** - * Determines how many bytes occupies a given data type. - * - * @param dataType data type to analyze - * @return number of bytes or None, if it's impossible to determine a priori - */ - def dataTypeByteSize(dataType: DataType): Option[Int] = { - dataType match { - case _: Int1Type => Some(1) - case IntMultiType(_, width, _) => Some(width.width) - case FixedBytesType(contents, _) => Some(contents.length) - case FloatMultiType(width, _) => Some(width.width) - case _: BytesEosType => None - case blt: BytesLimitType => evaluateIntLiteral(blt.size) - case _: BytesTerminatedType => None - case StrFromBytesType(basedOn, _) => dataTypeByteSize(basedOn) - case utb: UserTypeFromBytes => dataTypeByteSize(utb.bytes) - case UserTypeInstream(_, _) => None - case _: SwitchType => None - } - } - def dataTypeName(dataType: DataType): String = { dataType match { - case rt: ReadableType => rt.apiCall + case rt: ReadableType => rt.apiCall(None) // FIXME case ut: UserType => type2display(ut.name) case FixedBytesType(contents, _) => contents.map(_.formatted("%02X")).mkString(" ") case BytesTerminatedType(terminator, include, consume, eosError, _) => @@ -462,21 +431,25 @@ object GraphvizClassCompiler extends LanguageCompilerStatic { } } + def htmlEscape(s: String): String = { + s.replaceAll("&", "&").replaceAll("<", "<").replaceAll(">", ">").replaceAll("\"", """) + } + /** - * Evaluates the expression, if possible to get the result without introduction - * of any variables or anything. - * - * @param expr expression to evaluate - * @return integer result or None + * Converts bit-level position into byte/bit human-readable combination. + * @param seqPos optional number of bits + * @return fractional human-readable string which displays "bytes:bits", + * akin to "minutes:seconds" time display */ - def evaluateIntLiteral(expr: Ast.expr): Option[Int] = { - expr match { - case Ast.expr.IntNum(x) => Some(x.toInt) - case _ => None + def seqPosToStr(seqPos: Option[Int]): Option[String] = { + seqPos.map { (pos) => + val posByte = pos / 8 + val posBit = pos % 8 + if (posBit == 0) { + s"$posByte" + } else { + s"$posByte:$posBit" + } } } - - def htmlEscape(s: String): String = { - s.replaceAll("&", "&").replaceAll("<", "<").replaceAll(">", ">").replaceAll("\"", """) - } } diff --git a/shared/src/main/scala/io/kaitai/struct/HtmlClassCompiler.scala b/shared/src/main/scala/io/kaitai/struct/HtmlClassCompiler.scala new file mode 100644 index 000000000..9d9d74c1a --- /dev/null +++ b/shared/src/main/scala/io/kaitai/struct/HtmlClassCompiler.scala @@ -0,0 +1,152 @@ +package io.kaitai.struct + +import io.kaitai.struct.datatype.DataType +import io.kaitai.struct.datatype.DataType.UserType +import io.kaitai.struct.exprlang.Ast +import io.kaitai.struct.format._ +import io.kaitai.struct.languages.components.{LanguageCompiler, LanguageCompilerStatic} + +class HtmlClassCompiler(classSpecs: ClassSpecs, topClass: ClassSpec) extends DocClassCompiler(classSpecs, topClass) { + import HtmlClassCompiler._ + + override def outFileName(topClass: ClassSpec): String = s"${topClass.nameAsStr}.html" + + override def indent: String = "" + + override def fileHeader(topClass: ClassSpec): Unit = { + out.puts( + s""" + | + | + | + | + | + | + | + | + | + | + | ${type2str(topClass.name.last)} format specification + | + | +
+ |

${type2str(topClass.name.last)} format specification

+ | + """.stripMargin) + + // TODO: parse & output meta/title, meta/file-extensions, etc + } + + override def fileFooter(topClass: ClassSpec): Unit = { + out.puts( + """ + |
+ | + | + | + | + | + | + | + """.stripMargin) + } + + override def classHeader(classSpec: ClassSpec): Unit = { + out.puts(s"") + out.puts(s"<$headerByIndent>Type: ${type2str(classSpec.name.last)}") + out.puts + + classSpec.doc.summary.foreach(summary => + out.puts(s"

$summary

") + ) + out.inc + } + + override def classFooter(classSpec: ClassSpec): Unit = { + out.dec + } + + override def seqHeader(classSpec: ClassSpec): Unit = { + out.puts("") + out.puts("") + } + + override def seqFooter(classSpec: ClassSpec): Unit = { + out.puts("
OffsetSizeIDTypeNote
") + } + + override def compileSeqAttr(classSpec: ClassSpec, attr: AttrSpec, seqPos: Option[Int], sizeElement: Sized, sizeContainer: Sized): Unit = { + out.puts("") + out.puts(s"${GraphvizClassCompiler.seqPosToStr(seqPos).getOrElse("???")}") + out.puts(s"...") + out.puts(s"${attr.id.humanReadable}") + out.puts(s"${kaitaiType2NativeType(attr.dataType)}") + out.puts(s"${attr.doc.summary.getOrElse("")}") + out.puts("") + } + + override def compileParseInstance(classSpec: ClassSpec, inst: ParseInstanceSpec): Unit = { + out.puts(s"

Parse instance: ${inst.id.humanReadable}

") + out.puts("") + out.puts("") + out.puts(s"") + out.puts(s"") + out.puts(s"") + out.puts(s"") + out.puts(s"") + out.puts("") + out.puts("
${expression(inst.pos)}...${inst.id.humanReadable}${kaitaiType2NativeType(inst.dataType)}${inst.doc.summary.getOrElse("")}
") + } + + override def compileValueInstance(vis: ValueInstanceSpec): Unit = { + out.puts(s"value instance: ${vis}") + } + + override def compileEnum(enumName: String, enumColl: EnumSpec): Unit = { + out.puts(s"") + out.puts(s"<$headerByIndent>Enum: $enumName") + out.puts + + out.puts("") + out.puts("") + out.puts("") + out.puts("") + + enumColl.sortedSeq.foreach { case (id, value) => + out.puts("") + out.puts(s"") + out.puts("") + } + + out.puts("
IDNameNote
$id${value.name}${value.doc.summary.getOrElse("")}
") + } + + def headerByIndent: String = s"h${out.indentLevel + 1}" + + def expression(exOpt: Option[Ast.expr]): String = { + exOpt match { + case Some(ex) => translator.translate(ex) + case None => "" + } + } +} + +object HtmlClassCompiler extends LanguageCompilerStatic { + // FIXME: Unused, should be probably separated from LanguageCompilerStatic + override def getCompiler( + tp: ClassTypeProvider, + config: RuntimeConfig + ): LanguageCompiler = ??? + + def type2str(name: String): String = Utils.upperCamelCase(name) + + def classSpec2Anchor(spec: ClassSpec): String = "type-" + spec.name.mkString("-") + + def enumSpec2Anchor(spec: EnumSpec): String = "enum-" + spec.name.mkString("-") + + def kaitaiType2NativeType(attrType: DataType): String = attrType match { + case ut: UserType => + "" + type2str(ut.name.last) + "" + case _ => GraphvizClassCompiler.dataTypeName(attrType) + } +} diff --git a/shared/src/main/scala/io/kaitai/struct/ImportList.scala b/shared/src/main/scala/io/kaitai/struct/ImportList.scala new file mode 100644 index 000000000..b619100d1 --- /dev/null +++ b/shared/src/main/scala/io/kaitai/struct/ImportList.scala @@ -0,0 +1,13 @@ +package io.kaitai.struct + +import scala.collection.mutable.ListBuffer + +/** + * Manages imports/includes/requires/etc lists used for particular compilation + * unit, makes sure they are unique. + */ +class ImportList { + private val list = ListBuffer[String]() + def add(s: String) = Utils.addUniqueAttr(list, s) + def toList: List[String] = list.toList +} diff --git a/shared/src/main/scala/io/kaitai/struct/JSON.scala b/shared/src/main/scala/io/kaitai/struct/JSON.scala index cc3fbc7d9..adb3fc04c 100644 --- a/shared/src/main/scala/io/kaitai/struct/JSON.scala +++ b/shared/src/main/scala/io/kaitai/struct/JSON.scala @@ -1,16 +1,17 @@ package io.kaitai.struct -import io.kaitai.struct.format.ClassSpec -import io.kaitai.struct.translators.JavaScriptTranslator +import io.kaitai.struct.translators.CommonLiterals +/** Common trait for all objects that can be serialized as JSON. */ trait Jsonable { + /** Serialize current state of the object into JSON string. */ def toJson: String } /** * Ultra-minimalistic JSON strings generator from arbitrary Scala objects. */ -object JSON { +object JSON extends CommonLiterals { /** * Converts an arbitrary Scala object to JSON string representation. * @param obj object to convert @@ -26,12 +27,8 @@ object JSON { } } - private lazy val translator = new JavaScriptTranslator( - new ClassTypeProvider(ClassSpec.opaquePlaceholder(List("foo"))) - ) - def stringToJson(str: String): String = - translator.doStringLiteral(str) + doStringLiteral(str) def listToJson(obj: List[_]): String = "[" + obj.map((x) => stringify(x)).mkString(",") + "]" diff --git a/shared/src/main/scala/io/kaitai/struct/Log.scala b/shared/src/main/scala/io/kaitai/struct/Log.scala index 13ad6689b..1aeb161f6 100644 --- a/shared/src/main/scala/io/kaitai/struct/Log.scala +++ b/shared/src/main/scala/io/kaitai/struct/Log.scala @@ -25,6 +25,8 @@ object Log { "value", "parent", "type_resolve", + "type_valid", + "seq_sizes", "import" ) @@ -32,6 +34,8 @@ object Log { var typeProcValue: Logger = NullLogger var typeProcParent: Logger = NullLogger var typeResolve: Logger = NullLogger + var typeValid: Logger = NullLogger + var seqSizes: Logger = NullLogger var importOps: Logger = NullLogger def initFromVerboseFlag(subsystems: Seq[String]): Unit = { @@ -43,6 +47,8 @@ object Log { case "value" => typeProcValue = ConsoleLogger case "parent" => typeProcParent = ConsoleLogger case "type_resolve" => typeResolve = ConsoleLogger + case "type_valid" => typeValid = ConsoleLogger + case "seq_sizes" => seqSizes = ConsoleLogger case "import" => importOps = ConsoleLogger } } diff --git a/shared/src/main/scala/io/kaitai/struct/Main.scala b/shared/src/main/scala/io/kaitai/struct/Main.scala index c43a0e14c..daf53bc8b 100644 --- a/shared/src/main/scala/io/kaitai/struct/Main.scala +++ b/shared/src/main/scala/io/kaitai/struct/Main.scala @@ -1,6 +1,7 @@ package io.kaitai.struct import io.kaitai.struct.format.{ClassSpec, ClassSpecs, GenericStructClassSpec} +import io.kaitai.struct.languages.{GoCompiler, RustCompiler} import io.kaitai.struct.languages.components.LanguageCompilerStatic import io.kaitai.struct.precompile._ @@ -20,7 +21,7 @@ object Main { * into it and modifying classes itself by precompilation step */ def importAndPrecompile(specs: ClassSpecs, config: RuntimeConfig): Future[Unit] = { - new LoadImports(specs).processClass(specs.firstSpec).map { (allSpecs) => + new LoadImports(specs).processClass(specs.firstSpec, LoadImports.BasePath).map { (allSpecs) => Log.importOps.info(() => s"imports done, got: ${specs.keys} (async=$allSpecs)") specs.foreach { case (_, classSpec) => @@ -31,11 +32,12 @@ object Main { def precompile(classSpecs: ClassSpecs, topClass: ClassSpec, config: RuntimeConfig): Unit = { classSpecs.foreach { case (_, curClass) => MarkupClassNames.markupClassNames(curClass) } - val opaqueTypes = topClass.meta.get.opaqueTypes.getOrElse(config.opaqueTypes) + val opaqueTypes = topClass.meta.opaqueTypes.getOrElse(config.opaqueTypes) new ResolveTypes(classSpecs, opaqueTypes).run() - classSpecs.foreach { case (_, curClass) => ParentTypes.markup(curClass) } + new ParentTypes(classSpecs).run() new SpecsValueTypeDerive(classSpecs).run() - new TypeValidator(topClass).run() + new CalculateSeqSizes(classSpecs).run() + new TypeValidator(classSpecs, topClass).run() topClass.parentClass = GenericStructClassSpec } @@ -43,19 +45,30 @@ object Main { /** * Compiles a single [[ClassSpec]] into a single target language using * provided configuration. + * @param specs bundle of class specifications (used to search to references there) * @param spec class specification to compile * @param lang specifies which language compiler will be used * @param conf runtime compiler configuration * @return a container that contains all compiled files and results */ - def compile(spec: ClassSpec, lang: LanguageCompilerStatic, conf: RuntimeConfig): CompileLog.SpecSuccess = { + def compile(specs: ClassSpecs, spec: ClassSpec, lang: LanguageCompilerStatic, conf: RuntimeConfig): CompileLog.SpecSuccess = { val config = updateConfig(conf, spec) val cc = lang match { case GraphvizClassCompiler => - new GraphvizClassCompiler(spec) + new GraphvizClassCompiler(specs, spec) + case GoCompiler => + new GoClassCompiler(specs, spec, config) + case RustCompiler => + new RustClassCompiler(specs, spec, config) + case ConstructClassCompiler => + new ConstructClassCompiler(specs, spec) + case HtmlClassCompiler => + new HtmlClassCompiler(specs, spec) + case NimClassCompiler => + new NimClassCompiler(specs, spec, config) case _ => - new ClassCompiler(spec, config, lang) + new ClassCompiler(specs, spec, config, lang) } cc.compile } @@ -68,8 +81,8 @@ object Main { * @return updated runtime configuration with applied enforcements */ private def updateConfig(config: RuntimeConfig, topClass: ClassSpec): RuntimeConfig = { - if (topClass.meta.get.forceDebug) { - config.copy(debug = true) + if (topClass.meta.forceDebug) { + config.copy(autoRead = false, readStoresPos = true) } else { config } diff --git a/shared/src/main/scala/io/kaitai/struct/NimClassCompiler.scala b/shared/src/main/scala/io/kaitai/struct/NimClassCompiler.scala new file mode 100644 index 000000000..48cc48eb4 --- /dev/null +++ b/shared/src/main/scala/io/kaitai/struct/NimClassCompiler.scala @@ -0,0 +1,299 @@ +package io.kaitai.struct + +import io.kaitai.struct.datatype.{DataType, Endianness, FixedEndian, InheritedEndian} +import io.kaitai.struct.datatype.DataType._ +import io.kaitai.struct.exprlang.Ast +import io.kaitai.struct.format._ +import io.kaitai.struct.languages.components.{LanguageCompiler, LanguageCompilerStatic} +import io.kaitai.struct.translators.NimTranslator + +class NimClassCompiler( + classSpecs: ClassSpecs, + topClass: ClassSpec, + config: RuntimeConfig +) extends AbstractCompiler { + import NimClassCompiler._ + + val out = new StringLanguageOutputWriter(indent) + val provider = new ClassTypeProvider(classSpecs, topClass) + val importList = new ImportList + val globalPragmaList = new ImportList + val translator = new NimTranslator(provider, importList) + + override def compile: CompileLog.SpecSuccess = { + importList.add(config.nimModule) + out.puts("type") + out.inc + compileTypes(topClass) + out.dec + out.puts + compileProcs(topClass) + + + CompileLog.SpecSuccess( + "", + List(CompileLog.FileSuccess( + outFileName(topClass.nameAsStr), + imports + "\n\n" + globalPragmas + "\n\n" + out.result + )) + ) + } + + def indent: String = " " + def outFileName(topClassName: String): String = s"$topClassName.nim" + def imports = + importList.toList.map((x) => s"import $x").mkString("\n") + + def globalPragmas = + globalPragmaList.toList.map((x) => s"{.$x.}").mkString("\n") + + def compileTypes(curClass: ClassSpec): Unit = { + compileSubtypes(curClass) + + val t = listToNim(curClass.name) + out.puts(s"${t}* = ref ${t}Obj") + out.puts(s"${t}Obj* = object") + out.inc + + val extraAttrs = List( + AttrSpec(List(), IoIdentifier, KaitaiStreamType), + AttrSpec(List(), RootIdentifier, UserTypeInstream(topClass.name, None)), + AttrSpec(List(), ParentIdentifier, curClass.parentType) + ) + + (extraAttrs ++ curClass.seq).foreach { + (attr) => { + val i = idToStr(attr.id) + val t = ksToNim(attr.dataTypeComposite) + out.puts(s"${i}${if (attr.id == IoIdentifier) "" else "*" }: $t") + } + } + + if (curClass.instances.size != 0) { + importList.add("options") + globalPragmaList.add("experimental: \"dotOperators\"") + } + + curClass.instances.foreach { + case (id, spec) => { + val i = idToStr(id) + val t = ksToNim(spec.dataTypeComposite) + out.puts(s"${i}: proc(): $t") + } + } + out.dec + } + + def compileSubtypes(curClass: ClassSpec): Unit = { + curClass.types.foreach { case (_, subClass) => compileTypes(subClass) } + } + + def compileProcs(curClass: ClassSpec): Unit = { + compileSubtypeProcs(curClass) + + val t = listToNim(curClass.name) + val p = ksToNim(curClass.parentType) + val r = camelCase(topClass.name.head, true) + + out.puts(s"# $t") + + if (curClass.instances.size != 0) { + out.puts(s"template `.`*(a: $t, b: untyped): untyped =") + out.inc + out.puts("(a.`b inst`)()") + out.dec + out.puts + } + + out.puts(s"proc read*(_: typedesc[$t], io: KaitaiStream, root: $r, parent: $p): owned $t =") + out.inc + out.puts(s"result = new($t)") + out.puts(s"let root = if root == nil: cast[$r](result) else: root") + out.puts("result.io = io") + out.puts("result.root = root") + out.puts("result.parent = parent") + out.puts + var wasUnaligned = false + curClass.seq.foreach { + (attr) => { + val nowUnaligned = isUnalignedBits(attr.dataType) + if (wasUnaligned && !nowUnaligned) + out.puts("alignToByte(io)") + val i = idToStr(attr.id) + val dt = attr.dataTypeComposite + // XXX: fix endian + out.puts(s"let $i = ${parse(dt, "io", None)}") + out.puts(s"result.$i = $i") + wasUnaligned = nowUnaligned + } + } + out.puts + + curClass.instances.foreach { + case (id, spec) => { + val i = idToStr(id) + val r = i.dropRight(4) + val v = i.dropRight(4) + "Val" + val t = ksToNim(spec.dataTypeComposite) + out.puts(s"var ${v}: Option[$t]") + out.puts(s"let $r = proc(): $t =") + out.inc + out.puts(s"if isNone($v):") + out.inc + spec match { + case s: ValueInstanceSpec => { + s.dataType match { + case Some(dt) => out.puts(s"$v = some(${ksToNim(dt)}(${translator.translate(s.value)}))") + case None => out.puts(s"$v = some(${translator.translate(s.value)})") + } + + } + case s: ParseInstanceSpec => { + out.puts(s"$v = ${parse(s.dataType, "io", None)}") + } + } + out.dec + out.puts(s"get($v)") + out.dec + out.puts(s"result.$i = $r") + } + } + out.dec + out.puts + out.puts(s"proc fromFile*(_: typedesc[$t], filename: string): owned $t =") + out.inc + out.puts(s"$t.read(newKaitaiStream(filename), nil, nil)") + out.dec + out.puts + out.puts(s"proc `=destroy`(x: var ${t}Obj) =") + out.inc + out.puts("close(x.io)") + out.dec + out.puts + } + + def compileSubtypeProcs(curClass: ClassSpec): Unit = { + curClass.types.foreach { case (_, subClass) => compileProcs(subClass) } + } + + def parse(dataType: DataType, io: String, endian: Option[FixedEndian]): String = { + def process(unproc: String, proc: Option[ProcessExpr]): String = + proc match { + case None => unproc + case Some(proc) => + proc match { + case ProcessXor(key) => unproc + s".processXor(${evalLocal(key)})" + } + } + + dataType match { + case t: ReadableType => + s"read${Utils.capitalize(t.apiCall(endian))}($io)" + case t: BytesLimitType => + process(s"readBytes($io, int(${evalLocal(t.size)}))", t.process) + case t: BytesEosType => + process(s"readBytesFull($io)", t.process) + case BytesTerminatedType(terminator, include, consume, eosError, proc) => + process(s"readBytesTerm($io, $terminator, $include, $consume, $eosError)", proc) + case BitsType1 => + s"bool(readBitsInt($io, 1))" + case BitsType(width: Int) => + s"readBitsInt($io, $width)" + case t: UserTypeFromBytes => + parse(t.bytes, "io", None) + case t: UserType => + val addArgs = if (t.isOpaque) { + "" + } else { + val parent = t.forcedParent match { + case Some(USER_TYPE_NO_PARENT) => "nil" + case Some(fp) => translator.translate(fp) + case None => "result" // ??? + } + s", root, $parent" + } + s"${listToNim(t.name)}.read($io$addArgs)" + } + } + + def evalLocal(e: Ast.expr): String = + s"${e match {case Ast.expr.Name(_) => "result." case _ => ""}}${translator.translate(e)}" + + def isUnalignedBits(dt: DataType): Boolean = + dt match { + case _: BitsType | BitsType1 => true + case et: EnumType => isUnalignedBits(et.basedOn) + case _ => false + } +} + +object NimClassCompiler extends LanguageCompilerStatic { + override def getCompiler( + tp: ClassTypeProvider, + config: RuntimeConfig + ): LanguageCompiler = ??? + + def ksToNim(attrType: DataType): String = { + attrType match { + case Int1Type(false) => "uint8" + case IntMultiType(false, Width2, _) => "uint16" + case IntMultiType(false, Width4, _) => "uint32" + case IntMultiType(false, Width8, _) => "uint64" + + case Int1Type(true) => "int8" + case IntMultiType(true, Width2, _) => "int16" + case IntMultiType(true, Width4, _) => "int32" + case IntMultiType(true, Width8, _) => "int64" + + case FloatMultiType(Width4, _) => "float32" + case FloatMultiType(Width8, _) => "float64" + + case BitsType(_) => "uint64" + + case _: BooleanType => "bool" + case CalcIntType => "int" + case CalcFloatType => "float64" + + case _: StrType => "string" + case _: BytesType => "seq[byte]" + + case KaitaiStructType | CalcKaitaiStructType => "ref RootObj" + case KaitaiStreamType => "KaitaiStream" + + case t: UserType => listToNim(t.name) + case EnumType(name, _) => listToNim(name) + + case ArrayType(inType) => s"seq[${ksToNim(inType)}]" + + case st: SwitchType => ksToNim(st.combinedType) + } + } + + def camelCase(s: String, upper: Boolean): String = { + if (upper) { + s.split("_").map(Utils.capitalize).mkString + } else { + if (s.startsWith("_")) { + camelCase(s.substring(1), false) + } else { + val firstWord :: restWords = s.split("_").toList + (firstWord :: restWords.map(Utils.capitalize)).mkString + } + } + } + + def idToStr(id: Identifier): String = { + id match { + case IoIdentifier => "io" + case NamedIdentifier(name) => camelCase(name, false) + case InstanceIdentifier(name) => camelCase(name, false) + "Inst" + case IoStorageIdentifier(innerId) => "io" + camelCase(idToStr(innerId), true) + + case SpecialIdentifier(name) => camelCase(name, false) + case NumberedIdentifier(idx) => s"${NumberedIdentifier.TEMPLATE}$idx" + case RawIdentifier(innerId) => "raw" + camelCase(idToStr(innerId), true) + } + } + + def listToNim(names: List[String]) = camelCase(names.last, true) +} \ No newline at end of file diff --git a/shared/src/main/scala/io/kaitai/struct/RuntimeConfig.scala b/shared/src/main/scala/io/kaitai/struct/RuntimeConfig.scala index 42bb37501..0ac05dc14 100644 --- a/shared/src/main/scala/io/kaitai/struct/RuntimeConfig.scala +++ b/shared/src/main/scala/io/kaitai/struct/RuntimeConfig.scala @@ -1,10 +1,84 @@ package io.kaitai.struct +/** + * C++-specific runtime configuration of the compiler. + * @param usePragmaOnce If true, use `#pragma once` in headers. If false (default), + * use `#ifndef`-`#define`-`#endif` guards. + * @param stdStringFrontBack If true, allow use of `front()` and `back()` methods + * on `std::string`. If false, come up with simulation. + * @param pointers Choose which style of pointers to use. + */ +case class CppRuntimeConfig( + namespace: List[String] = List(), + usePragmaOnce: Boolean = false, + stdStringFrontBack: Boolean = false, + pointers: CppRuntimeConfig.Pointers = CppRuntimeConfig.RawPointers +) { + /** + * Copies this C++ runtime config, applying all the default settings for + * C++98 target. + */ + def copyAsCpp98() = copy( + usePragmaOnce = false, + stdStringFrontBack = false, + pointers = CppRuntimeConfig.RawPointers + ) + + /** + * Copies this C++ runtime config, applying all the default settings for + * C++11 target. + */ + def copyAsCpp11() = copy( + usePragmaOnce = true, + stdStringFrontBack = true, + pointers = CppRuntimeConfig.UniqueAndRawPointers + ) +} + +object CppRuntimeConfig { + sealed trait Pointers + case object RawPointers extends Pointers + case object SharedPointers extends Pointers + case object UniqueAndRawPointers extends Pointers +} + +/** + * Runtime configuration of the compiler which controls certain aspects of + * code generation for target languages. + * @param autoRead If true, constructor (or equivalent) invocation would + * automatically run `_read` (or equivalent), thus allowing to + * run parsing just by constructing an object, passing a stream + * into it. If false, `_read` would be made public and it is + * expected to be invoked manually. + * @param readStoresPos If true, parser (`_read` or equivalent) will store + * positions of all the attributes relative to the stream; + * not required for production usage (as it is typically slow + * and memory-consuming), but it is crucial for visualizers, + * IDEs, etc, to be able to display data layout. + * @param opaqueTypes If true, invoking any unknown type will be treated as it was + * "opaque" type, i.e. an external KaitaiStruct-compatible type + * defined somewhere else. If false, it will be reported as + * precompile error. + * @param cppConfig C++-specific configuration + * @param goPackage Go package name + * @param javaPackage Java package name + * @param javaFromFileClass Java class to be invoked in `fromFile` helper methods + * @param dotNetNamespace .NET (C#) namespace + * @param phpNamespace PHP namespace + * @param pythonPackage Python package name + * @param nimModule Path of Nim runtime module + */ case class RuntimeConfig( - debug: Boolean = false, + autoRead: Boolean = true, + readStoresPos: Boolean = false, opaqueTypes: Boolean = false, readWrite: Boolean = false, + cppConfig: CppRuntimeConfig = CppRuntimeConfig(), + goPackage: String = "", javaPackage: String = "", + javaFromFileClass: String = "io.kaitai.struct.ByteBufferKaitaiStream", dotNetNamespace: String = "Kaitai", - phpNamespace: String = "" + phpNamespace: String = "", + pythonPackage: String = "", + nimModule: String = "kaitai_struct/runtime/nim/kaitai" ) diff --git a/shared/src/main/scala/io/kaitai/struct/RustClassCompiler.scala b/shared/src/main/scala/io/kaitai/struct/RustClassCompiler.scala new file mode 100644 index 000000000..8deaedc7f --- /dev/null +++ b/shared/src/main/scala/io/kaitai/struct/RustClassCompiler.scala @@ -0,0 +1,104 @@ +package io.kaitai.struct + +import io.kaitai.struct.datatype.DataType.{KaitaiStreamType, UserTypeInstream} +import io.kaitai.struct.datatype.{Endianness, FixedEndian, InheritedEndian} +import io.kaitai.struct.format._ +import io.kaitai.struct.languages.RustCompiler +import io.kaitai.struct.languages.components.ExtraAttrs + +import scala.collection.mutable.ListBuffer + +class RustClassCompiler( + classSpecs: ClassSpecs, + override val topClass: ClassSpec, + config: RuntimeConfig +) extends ClassCompiler(classSpecs, topClass, config, RustCompiler) { + + override def compileClass(curClass: ClassSpec): Unit = { + provider.nowClass = curClass + + val extraAttrs = ListBuffer[AttrSpec]() + extraAttrs += AttrSpec(List(), IoIdentifier, KaitaiStreamType) + extraAttrs += AttrSpec(List(), RootIdentifier, UserTypeInstream(topClassName, None)) + extraAttrs += AttrSpec(List(), ParentIdentifier, curClass.parentType) + + extraAttrs ++= ExtraAttrs.forClassSpec(curClass, lang) + + if (!curClass.doc.isEmpty) + lang.classDoc(curClass.name, curClass.doc) + + // Basic struct declaration + lang.classHeader(curClass.name) + + compileAttrDeclarations(curClass.seq ++ extraAttrs) + curClass.instances.foreach { case (instName, instSpec) => + compileInstanceDeclaration(instName, instSpec) + } + + // Constructor = Read() function + compileReadFunction(curClass) + + compileInstances(curClass) + + compileAttrReaders(curClass.seq ++ extraAttrs) + lang.classFooter(curClass.name) + + compileEnums(curClass) + + // Recursive types + compileSubclasses(curClass) + } + + def compileReadFunction(curClass: ClassSpec) = { + lang.classConstructorHeader( + curClass.name, + curClass.parentType, + topClassName, + curClass.meta.endian.contains(InheritedEndian), + curClass.params + ) + + // FIXME + val defEndian = curClass.meta.endian match { + case Some(fe: FixedEndian) => Some(fe) + case _ => None + } + + lang.readHeader(defEndian, false) + + compileSeqRead(curClass.seq, defEndian) + lang.classConstructorFooter + } + + override def compileInstances(curClass: ClassSpec) = { + lang.instanceDeclHeader(curClass.name) + curClass.instances.foreach { case (instName, instSpec) => + compileInstance(curClass.name, instName, instSpec, curClass.meta.endian) + } + } + + override def compileInstance(className: List[String], instName: InstanceIdentifier, instSpec: InstanceSpec, endian: Option[Endianness]): Unit = { + // FIXME: support calculated endianness + + // Determine datatype + val dataType = instSpec.dataTypeComposite + + if (!instSpec.doc.isEmpty) + lang.attributeDoc(instName, instSpec.doc) + lang.instanceHeader(className, instName, dataType, instSpec.isNullable) + lang.instanceCheckCacheAndReturn(instName, dataType) + + instSpec match { + case vi: ValueInstanceSpec => + lang.attrParseIfHeader(instName, vi.ifExpr) + lang.instanceCalculate(instName, dataType, vi.value) + lang.attrParseIfFooter(vi.ifExpr) + case i: ParseInstanceSpec => + lang.attrParse(i, instName, None) // FIXME + } + + lang.instanceSetCalculated(instName) + lang.instanceReturn(instName, dataType) + lang.instanceFooter + } +} diff --git a/shared/src/main/scala/io/kaitai/struct/TypeProcessor.scala b/shared/src/main/scala/io/kaitai/struct/TypeProcessor.scala index 7aaafaae4..92314ae9d 100644 --- a/shared/src/main/scala/io/kaitai/struct/TypeProcessor.scala +++ b/shared/src/main/scala/io/kaitai/struct/TypeProcessor.scala @@ -36,8 +36,8 @@ object TypeProcessor { } else { List() } - case SwitchType(_, cases) => - cases.flatMap { case (_, ut) => + case st: SwitchType => + st.cases.flatMap { case (_, ut) => getOpaqueDataTypes(ut) } case _ => diff --git a/shared/src/main/scala/io/kaitai/struct/Utils.scala b/shared/src/main/scala/io/kaitai/struct/Utils.scala index eab0ca8f5..bae0c7050 100644 --- a/shared/src/main/scala/io/kaitai/struct/Utils.scala +++ b/shared/src/main/scala/io/kaitai/struct/Utils.scala @@ -5,6 +5,11 @@ import java.nio.charset.Charset import scala.collection.mutable.ListBuffer object Utils { + /** + * BigInt-typed max value of unsigned 64-bit integer. + */ + val MAX_UINT64 = BigInt("18446744073709551615") + private val RDecimal = "^(-?[0-9]+)$".r private val RHex = "^0x([0-9a-fA-F]+)$".r @@ -64,6 +69,18 @@ object Utils { } } + /** + * Joins collection together to make a single string. Makes extra exception for empty + * collections (not like [[TraversableOnce]] `mkString`). + * @param start the starting string. + * @param sep the separator string. + * @param end the ending string. + * @return If the collection is empty, returns empty string, otherwise returns `start`, + * then elements of the collection, every pair separated with `sep`, then `end`. + */ + def join[T](coll: TraversableOnce[T], start: String, sep: String, end: String): String = + if (coll.isEmpty) "" else coll.mkString(start, sep, end) + /** * Converts byte array (Seq[Byte]) into hex-escaped C-style literal characters * (i.e. like \xFF). @@ -96,4 +113,23 @@ object Utils { } else { fullPath } + + /** + * Performs safe lookup for up to `len` character in a given + * string `src`, starting at `from`. + * @param src string to work on + * @param from starting character index + * @param len max length of substring + * @return substring of `src`, starting at `from`, up to `len` chars max + */ + def safeLookup(src: String, from: Int, len: Int): String = { + val maxLen = src.length + if (from >= maxLen) { + "" + } else { + val to = from + len + val safeTo = if (to > maxLen) maxLen else to + src.substring(from, safeTo) + } + } } diff --git a/shared/src/main/scala/io/kaitai/struct/datatype/DataType.scala b/shared/src/main/scala/io/kaitai/struct/datatype/DataType.scala index 85b2dd85d..f0abf6e80 100644 --- a/shared/src/main/scala/io/kaitai/struct/datatype/DataType.scala +++ b/shared/src/main/scala/io/kaitai/struct/datatype/DataType.scala @@ -1,9 +1,16 @@ package io.kaitai.struct.datatype -import io.kaitai.struct.exprlang.Ast +import io.kaitai.struct.exprlang.{Ast, Expressions} import io.kaitai.struct.format._ +import io.kaitai.struct.translators.TypeDetector -sealed trait DataType +sealed trait DataType { + /** + * @return Data type as non-owning data type. Default implementation + * always returns itself, complex types + */ + def asNonOwning: DataType = this +} /** * A collection of case objects and classes that are used to represent internal @@ -20,22 +27,23 @@ object DataType { * A common trait for all types that can be read with a simple, * parameterless KaitaiStream API call. */ - trait ReadableType extends DataType { - def apiCall: String + sealed trait ReadableType extends DataType { + def apiCall(defEndian: Option[FixedEndian]): String } - abstract class NumericType extends DataType - abstract class BooleanType extends DataType + abstract sealed class NumericType extends DataType + abstract sealed class BooleanType extends DataType - abstract class IntType extends NumericType + abstract sealed class IntType extends NumericType case object CalcIntType extends IntType case class Int1Type(signed: Boolean) extends IntType with ReadableType { - override def apiCall: String = if (signed) "s1" else "u1" + override def apiCall(defEndian: Option[FixedEndian]): String = if (signed) "s1" else "u1" } - case class IntMultiType(signed: Boolean, width: IntWidth, endian: Endianness) extends IntType with ReadableType { - override def apiCall: String = { + case class IntMultiType(signed: Boolean, width: IntWidth, endian: Option[FixedEndian]) extends IntType with ReadableType { + override def apiCall(defEndian: Option[FixedEndian]): String = { val ch1 = if (signed) 's' else 'u' - s"$ch1${width.width}${endian.toString}" + val finalEnd = endian.orElse(defEndian) + s"$ch1${width.width}${finalEnd.map(_.toSuffix).getOrElse("")}" } } case object BitsType1 extends BooleanType @@ -43,9 +51,10 @@ object DataType { abstract class FloatType extends NumericType case object CalcFloatType extends FloatType - case class FloatMultiType(width: IntWidth, endian: Endianness) extends FloatType with ReadableType { - override def apiCall: String = { - s"f${width.width}${endian.toString}" + case class FloatMultiType(width: IntWidth, endian: Option[FixedEndian]) extends FloatType with ReadableType { + override def apiCall(defEndian: Option[FixedEndian]): String = { + val finalEnd = endian.orElse(defEndian) + s"f${width.width}${finalEnd.map(_.toSuffix).getOrElse("")}" } } @@ -84,53 +93,212 @@ object DataType { case class StrFromBytesType(bytes: BytesType, encoding: String) extends StrType case object CalcBooleanType extends BooleanType - case class ArrayType(elType: DataType) extends DataType - abstract class UserType(val name: List[String], val forcedParent: Option[Ast.expr]) extends DataType { + /** + * Complex data type is a data type which creation and destruction is + * not an atomic, built-in operation, but rather a sequence of new/delete + * operations. The main common trait for all complex data types is a flag + * that determines whether they're "owning" or "borrowed". Owning objects + * manage their own creation/destruction, borrowed rely on other doing + * that. + */ + abstract sealed class ComplexDataType extends DataType { + /** + * @return If true, this is "owning" type: for languages where data ownership + * matters, this one represents primary owner of the data block, who + * will be responsible for whole life cycle: creation of the object + * and its destruction. + */ + def isOwning: Boolean + } + + /** + * Common abstract ancestor for all types which can treated as "user types". + * Namely, this typically means that this type has a name, may have some + * parameters, and forced parent expression. + * @param name name of the type, might include several components + * @param forcedParent optional parent enforcement expression + * @param args parameters passed into this type as extra arguments + */ + abstract class UserType( + val name: List[String], + val forcedParent: Option[Ast.expr], + var args: Seq[Ast.expr] + ) extends ComplexDataType { var classSpec: Option[ClassSpec] = None def isOpaque = { val cs = classSpec.get - cs.isTopLevel || (cs.meta match { - case None => false - case Some(meta) => meta.isOpaque - }) + cs.isTopLevel || cs.meta.isOpaque + } + + override def asNonOwning: UserType = { + if (!isOwning) { + this + } else { + val r = CalcUserType(name, forcedParent, args) + r.classSpec = classSpec + r + } } } - case class UserTypeInstream(_name: List[String], _forcedParent: Option[Ast.expr]) extends UserType(_name, _forcedParent) + case class UserTypeInstream( + _name: List[String], + _forcedParent: Option[Ast.expr], + _args: Seq[Ast.expr] = Seq() + ) extends UserType(_name, _forcedParent, _args) { + def isOwning = true + } case class UserTypeFromBytes( _name: List[String], _forcedParent: Option[Ast.expr], + _args: Seq[Ast.expr] = Seq(), bytes: BytesType, override val process: Option[ProcessExpr] - ) extends UserType(_name, _forcedParent) with Processing + ) extends UserType(_name, _forcedParent, _args) with Processing { + override def isOwning = true + } + case class CalcUserType( + _name: List[String], + _forcedParent: Option[Ast.expr], + _args: Seq[Ast.expr] = Seq() + ) extends UserType(_name, _forcedParent, _args) { + override def isOwning = false + } + + case class ArrayType(elType: DataType) extends ComplexDataType { + override def isOwning: Boolean = true + override def asNonOwning: CalcArrayType = CalcArrayType(elType) + } + case class CalcArrayType(elType: DataType) extends ComplexDataType { + override def isOwning: Boolean = false + } val USER_TYPE_NO_PARENT = Ast.expr.Bool(false) case object AnyType extends DataType - case object KaitaiStructType extends DataType + case object KaitaiStructType extends ComplexDataType { + def isOwning = true + override def asNonOwning: DataType = CalcKaitaiStructType + } + case object CalcKaitaiStructType extends ComplexDataType { + def isOwning = false + } case object KaitaiStreamType extends DataType case class EnumType(name: List[String], basedOn: IntType) extends DataType { var enumSpec: Option[EnumSpec] = None } - case class SwitchType(on: Ast.expr, cases: Map[Ast.expr, DataType]) extends DataType + case class SwitchType(on: Ast.expr, cases: Map[Ast.expr, DataType], isOwning: Boolean = true) extends ComplexDataType { + def combinedType: DataType = TypeDetector.combineTypes(cases.values) + + /** + * @return True if this switch type includes an "else" case + */ + def hasElseCase: Boolean = cases.contains(SwitchType.ELSE_CONST) + + /** + * If a switch type has no else statement, it will turn out to be null + * every case would fail, so it's nullable. + * @return True if this switch type is nullable for regular languages. + */ + def isNullable: Boolean = !hasElseCase + + /** + * @return True if this switch type is nullable in a raw switch bytes languages (C++). + */ + def isNullableSwitchRaw: Boolean = { + val elseCase = cases.get(SwitchType.ELSE_CONST) + elseCase match { + case Some(_: BytesType) => + // else case with bytes type, nullable for C++-like languages + true + case Some(x) => + // else case with any user type, non-nullable + false + case None => + // no else case, even raw bytes, definitely nullable + true + } + } + + def hasSize: Boolean = + cases.values.exists((t) => + t.isInstanceOf[UserTypeFromBytes] || t.isInstanceOf[BytesType] + ) + + override def asNonOwning: DataType = SwitchType(on, cases, false) + } object SwitchType { /** * Constant that would be used for "else" case in SwitchType case class "cases" map. */ val ELSE_CONST = Ast.expr.Name(Ast.identifier("_")) + + val LEGAL_KEYS_SWITCH = Set( + "switch-on", + "cases" + ) + + def fromYaml1(switchSpec: Map[String, Any], path: List[String]): (String, Map[String, String]) = { + val _on = ParseUtils.getValueStr(switchSpec, "switch-on", path) + val _cases: Map[String, String] = switchSpec.get("cases") match { + case None => Map() + case Some(x) => ParseUtils.asMapStrStr(x, path ++ List("cases")) + } + + ParseUtils.ensureLegalKeys(switchSpec, LEGAL_KEYS_SWITCH, path) + (_on, _cases) + } + + def fromYaml( + switchSpec: Map[String, Any], + path: List[String], + metaDef: MetaSpec, + arg: YamlAttrArgs + ): SwitchType = { + val (_on, _cases) = fromYaml1(switchSpec, path) + + val on = Expressions.parse(_on) + val cases: Map[Ast.expr, DataType] = _cases.map { case (condition, typeName) => + Expressions.parse(condition) -> DataType.fromYaml( + Some(typeName), path ++ List("cases"), metaDef, + arg + ) + } + + // If we have size defined, and we don't have any "else" case already, add + // an implicit "else" case that will at least catch everything else as + // "untyped" byte array of given size + val addCases: Map[Ast.expr, DataType] = if (cases.contains(ELSE_CONST)) { + Map() + } else { + (arg.size, arg.sizeEos) match { + case (Some(sizeValue), false) => + Map(SwitchType.ELSE_CONST -> BytesLimitType(sizeValue, None, false, None, arg.process)) + case (None, true) => + Map(SwitchType.ELSE_CONST -> BytesEosType(None, false, None, arg.process)) + case (None, false) => + Map() + case (Some(_), true) => + throw new YAMLParseException("can't have both `size` and `size-eos` defined", path) + } + } + + SwitchType(on, cases ++ addCases) + } } private val ReIntType = """([us])(2|4|8)(le|be)?""".r private val ReFloatType = """f(4|8)(le|be)?""".r - private val ReBitType= """b(\d+)""".r + private val ReBitType = """b(\d+)""".r + private val ReUserTypeWithArgs = """(.+)\((.*)\)""".r def fromYaml( dto: Option[String], path: List[String], - metaDef: MetaDefaults, + metaDef: MetaSpec, arg: YamlAttrArgs ): DataType = { val r = dto match { @@ -185,19 +353,27 @@ object DataType { val bat = arg2.getByteArrayType(path) StrFromBytesType(bat, enc) case _ => - val dtl = classNameToList(dt) + val (arglessType, args) = dt match { + case ReUserTypeWithArgs(typeStr, argsStr) => (typeStr, Expressions.parseList(argsStr)) + case _ => (dt, List()) + } + val dtl = classNameToList(arglessType) if (arg.size.isEmpty && !arg.sizeEos && arg.terminator.isEmpty) { if (arg.process.isDefined) throw new YAMLParseException(s"user type '$dt': need 'size' / 'size-eos' / 'terminator' if 'process' is used", path) - UserTypeInstream(dtl, arg.parent) + UserTypeInstream(dtl, arg.parent, args) } else { val bat = arg.getByteArrayType(path) - UserTypeFromBytes(dtl, arg.parent, bat, arg.process) + UserTypeFromBytes(dtl, arg.parent, args, bat, arg.process) } } } - arg.enumRef match { + applyEnumType(r, arg.enumRef, path) + } + + private def applyEnumType(r: DataType, enumRef: Option[String], path: List[String]) = { + enumRef match { case Some(enumName) => r match { case numType: IntType => EnumType(classNameToList(enumName), numType) @@ -209,7 +385,56 @@ object DataType { } } - def getEncoding(curEncoding: Option[String], metaDef: MetaDefaults, path: List[String]): String = { + private val RePureIntType = """([us])(2|4|8)""".r + private val RePureFloatType = """f(4|8)""".r + + def pureFromString(dto: Option[String], enumRef: Option[String], path: List[String]): DataType = + applyEnumType(pureFromString(dto), enumRef, path) + + def pureFromString(dto: Option[String]): DataType = dto match { + case None => CalcBytesType + case Some(dt) => pureFromString(dt) + } + + def pureFromString(dt: String): DataType = dt match { + case "bytes" => CalcBytesType + case "u1" => Int1Type(false) + case "s1" => Int1Type(true) + case RePureIntType(signStr, widthStr) => + IntMultiType( + signStr match { + case "s" => true + case "u" => false + }, + widthStr match { + case "2" => Width2 + case "4" => Width4 + case "8" => Width8 + }, + None + ) + case RePureFloatType(widthStr) => + FloatMultiType( + widthStr match { + case "4" => Width4 + case "8" => Width8 + }, + None + ) + case ReBitType(widthStr) => + widthStr match { + case "1" => BitsType1 + case _ => BitsType(widthStr.toInt) + } + case "str" => CalcStrType + case "bool" => CalcBooleanType + case "struct" => CalcKaitaiStructType + case "io" => KaitaiStreamType + case "any" => AnyType + case _ => CalcUserType(classNameToList(dt), None) + } + + def getEncoding(curEncoding: Option[String], metaDef: MetaSpec, path: List[String]): String = { curEncoding.orElse(metaDef.encoding) match { case Some(enc) => enc case None => @@ -224,4 +449,4 @@ object DataType { * @return class name notation as list of components */ def classNameToList(s: String): List[String] = s.split("::", -1).toList -} \ No newline at end of file +} diff --git a/shared/src/main/scala/io/kaitai/struct/datatype/Endianness.scala b/shared/src/main/scala/io/kaitai/struct/datatype/Endianness.scala index b75150e1f..93cb5ff90 100644 --- a/shared/src/main/scala/io/kaitai/struct/datatype/Endianness.scala +++ b/shared/src/main/scala/io/kaitai/struct/datatype/Endianness.scala @@ -1,32 +1,67 @@ package io.kaitai.struct.datatype -import io.kaitai.struct.format.YAMLParseException +import io.kaitai.struct.datatype.DataType.SwitchType +import io.kaitai.struct.exprlang.{Ast, Expressions} +import io.kaitai.struct.format.{ParseUtils, YAMLParseException} sealed trait Endianness -case object LittleEndian extends Endianness { - override def toString = "le" + +sealed abstract class FixedEndian extends Endianness { + def toSuffix: String +} +case object LittleEndian extends FixedEndian { + override def toSuffix = "le" } -case object BigEndian extends Endianness { - override def toString = "be" +case object BigEndian extends FixedEndian { + override def toSuffix = "be" } +case class CalcEndian(on: Ast.expr, cases: Map[Ast.expr, FixedEndian]) extends Endianness + +case object InheritedEndian extends Endianness + object Endianness { - def defaultFromString(s: Option[String], path: List[String]) = s match { - case None => None - case Some("be") => Some(BigEndian) - case Some("le") => Some(LittleEndian) - case Some(unknown) => throw YAMLParseException.badDictValue( - Set("be", "le"), unknown, path ++ List("endian") - ) + def fromYaml(src: Option[Any], path: List[String]): Option[Endianness] = { + src match { + case None => None + case Some("be") => Some(BigEndian) + case Some("le") => Some(LittleEndian) + case Some(srcMap: Map[Any, Any]) => + val endianMap = ParseUtils.asMapStr(srcMap, path) + Some(fromMap(endianMap, path)) + case _ => + throw new YAMLParseException( + s"unable to parse endianness: `le`, `be` or calculated endianness map is expected", + path ++ List("endian") + ) + } } - def fromString(s: Option[String], defaultEndian: Option[Endianness], dt: String, path: List[String]) = s match { - case Some("le") => LittleEndian - case Some("be") => BigEndian - case None => + def fromMap(srcMap: Map[String, Any], path: List[String]): CalcEndian = { + val (_on, _cases) = SwitchType.fromYaml1(srcMap, path) + + val on = Expressions.parse(_on) + val cases: Map[Ast.expr, FixedEndian] = _cases.map { case (condition, endStr) => + Expressions.parse(condition) -> (endStr match { + case "be" => BigEndian + case "le" => LittleEndian + case _ => + throw YAMLParseException.badDictValue(Set("be", "le"), endStr, path ++ List("cases", condition)) + }) + } + + CalcEndian(on, cases) + } + + def fromString(s: Option[String], defaultEndian: Option[Endianness], dt: String, path: List[String]): Option[FixedEndian] = s match { + case Some("le") => Some(LittleEndian) + case Some("be") => Some(BigEndian) + case _ => defaultEndian match { - case Some(e) => e - case None => throw new YAMLParseException(s"unable to use type '$dt' without default endianness", path ++ List("type")) + case Some(e: FixedEndian) => Some(e) + case Some(_: CalcEndian) | Some(InheritedEndian) => None // to be overridden during compile + case None => + throw new YAMLParseException(s"unable to use type '$dt' without default endianness", path ++ List("type")) } } } diff --git a/shared/src/main/scala/io/kaitai/struct/datatype/KSError.scala b/shared/src/main/scala/io/kaitai/struct/datatype/KSError.scala new file mode 100644 index 000000000..4910d8ccd --- /dev/null +++ b/shared/src/main/scala/io/kaitai/struct/datatype/KSError.scala @@ -0,0 +1,44 @@ +package io.kaitai.struct.datatype + +/** + * Represents all errors & exceptions that Kaitai Struct-generated code + * in target languages could throw in runtime. + */ +sealed trait KSError { + def name: String +} + +object KSError { + val RE_VALIDATION_NOT_EQUAL = "^ValidationNotEqualError<(.*)>$".r + + def fromName(name: String): KSError = name match { + case "EndOfStreamError" => EndOfStreamError + case "UndecidedEndiannessError" => UndecidedEndiannessError + case RE_VALIDATION_NOT_EQUAL(dataTypeStr) => + ValidationNotEqualError(DataType.pureFromString(dataTypeStr)) + } +} + +/** + * Error to be thrown when validation on equality fails. + * @param dt data type used in validation process + */ +case class ValidationNotEqualError(dt: DataType) extends KSError { + def name = "ValidationNotEqualError" +} + +/** + * Exception that is thrown when we can't decided on endianness + * and thus can't proceed with parsing. + */ +case object UndecidedEndiannessError extends KSError { + def name = "UndecidedEndiannessError" +} + +/** + * Generic exception that is thrown when we're reached past + * the end of current stream. + */ +case object EndOfStreamError extends KSError { + def name = "EndOfStreamError" +} diff --git a/shared/src/main/scala/io/kaitai/struct/exprlang/Ast.scala b/shared/src/main/scala/io/kaitai/struct/exprlang/Ast.scala index ff23decd6..3f13b3105 100644 --- a/shared/src/main/scala/io/kaitai/struct/exprlang/Ast.scala +++ b/shared/src/main/scala/io/kaitai/struct/exprlang/Ast.scala @@ -21,9 +21,69 @@ package io.kaitai.struct.exprlang */ object Ast { case class identifier(name: String) + case class typeId(absolute: Boolean, names: Seq[String], isArray: Boolean = false) { + /** + * @return Type designation name as human-readable string, to be used in compiler + * error messages. + */ + def nameAsStr: String = + (if (absolute) "::" else "") + + names.mkString("::") + + (if (isArray) "[]" else "") + } + + val EmptyTypeId = typeId(false, Seq()) // BoolOp() can use left & right? - sealed trait expr + sealed trait expr { + /** + * Evaluates the expression, if it's possible to get a static integer + * constant as the result of evaluation (i.e. if it does not involve any + * variables or anything like that). Expect no complex logic or symbolic + * simplification of expressions here: something like "x - x", which is + * known to be always 0, will still report it as "None". + * + * @return integer result of evaluation if it's constant or None, if it's + * variable + */ + def evaluateIntConst: Option[BigInt] = { + this match { + case expr.IntNum(x) => + Some(x) + case expr.UnaryOp(op, operand) => + operand.evaluateIntConst.map(opValue => + op match { + case unaryop.Invert => ~opValue + case unaryop.Not => return None // TODO? + case unaryop.Minus => -opValue + } + ) + case expr.BinOp(left, op, right) => + val leftValue = left.evaluateIntConst match { + case Some(x) => x + case None => return None + } + val rightValue = right.evaluateIntConst match { + case Some(x) => x + case None => return None + } + op match { + case operator.Add => Some(leftValue + rightValue) + case operator.Sub => Some(leftValue - rightValue) + case operator.Mult => Some(leftValue * rightValue) + case operator.Div => Some(leftValue / rightValue) + case operator.Mod => Some(leftValue % rightValue) + case operator.LShift => Some(leftValue << rightValue.toInt) + case operator.RShift => Some(leftValue >> rightValue.toInt) + case operator.BitOr => Some(leftValue | rightValue) + case operator.BitXor => Some(leftValue ^ rightValue) + case operator.BitAnd => Some(leftValue & rightValue) + } + case _ => None + } + } + } + object expr{ case class BoolOp(op: boolop, values: Seq[expr]) extends expr case class BinOp(left: expr, op: operator, right: expr) extends expr @@ -36,11 +96,13 @@ object Ast { case class FloatNum(n: BigDecimal) extends expr case class Str(s: String) extends expr case class Bool(n: Boolean) extends expr - case class EnumByLabel(enumName: identifier, label: identifier) extends expr - case class EnumById(enumName: identifier, id: expr) extends expr + case class EnumByLabel(enumName: identifier, label: identifier, inType: typeId = EmptyTypeId) extends expr + case class EnumById(enumName: identifier, id: expr, inType: typeId = EmptyTypeId) extends expr case class Attribute(value: expr, attr: identifier) extends expr - case class CastToType(value: expr, typeName: identifier) extends expr + case class CastToType(value: expr, typeName: typeId) extends expr + case class ByteSizeOfType(typeName: typeId) extends expr + case class BitSizeOfType(typeName: typeId) extends expr case class Subscript(value: expr, idx: expr) extends expr case class Name(id: identifier) extends expr case class List(elts: Seq[expr]) extends expr diff --git a/shared/src/main/scala/io/kaitai/struct/exprlang/Expressions.scala b/shared/src/main/scala/io/kaitai/struct/exprlang/Expressions.scala index f0533ad29..ba47d1465 100644 --- a/shared/src/main/scala/io/kaitai/struct/exprlang/Expressions.scala +++ b/shared/src/main/scala/io/kaitai/struct/exprlang/Expressions.scala @@ -27,6 +27,10 @@ import fastparse.StringReprOps object Expressions { val NAME: P[Ast.identifier] = Lexical.identifier + val TYPE_NAME: P[Ast.typeId] = P("::".!.? ~ NAME.rep(1, "::") ~ ("[" ~ "]").!.?).map { + case (first, names: Seq[Ast.identifier], arrayStr) => + Ast.typeId(first.nonEmpty, names.map((el) => el.name), arrayStr.nonEmpty) + } val INT_NUMBER = Lexical.integer val FLOAT_NUMBER = Lexical.floatnumber val STRING: P[String] = Lexical.stringliteral @@ -43,7 +47,7 @@ object Expressions { case Seq(x) => x case xs => Ast.expr.BoolOp(Ast.boolop.And, xs) } - val not_test: P[Ast.expr] = P( ("not" ~ not_test).map(Ast.expr.UnaryOp(Ast.unaryop.Not, _)) | comparison ) + val not_test: P[Ast.expr] = P( (kw("not") ~ not_test).map(Ast.expr.UnaryOp(Ast.unaryop.Not, _)) | comparison ) val comparison: P[Ast.expr] = P( expr ~ (comp_op ~ expr).? ).map{ case (lhs, None) => lhs case (lhs, Some(chunks)) => @@ -114,6 +118,8 @@ object Expressions { "[" ~ list ~ "]" | // "{" ~ dictorsetmaker ~ "}" | enumByName | + byteSizeOfType | + bitSizeOfType | STRING.rep(1).map(_.mkString).map(Ast.expr.Str) | NAME.map((x) => x.name match { case "true" => Ast.expr.Bool(true) @@ -130,7 +136,7 @@ object Expressions { val trailer: P[Ast.expr => Ast.expr] = { val call = P("(" ~ arglist ~ ")").map{ case (args) => (lhs: Ast.expr) => Ast.expr.Call(lhs, args)} val slice = P("[" ~ test ~ "]").map{ case (args) => (lhs: Ast.expr) => Ast.expr.Subscript(lhs, args)} - val cast = P( "." ~ "as" ~ "<" ~ NAME ~ ">" ).map( + val cast = P( "." ~ "as" ~ "<" ~ TYPE_NAME ~ ">" ).map( typeName => (lhs: Ast.expr) => Ast.expr.CastToType(lhs, typeName) ) val attr = P("." ~ NAME).map(id => (lhs: Ast.expr) => Ast.expr.Attribute(lhs, id)) @@ -156,19 +162,39 @@ object Expressions { val testlist1: P[Seq[Ast.expr]] = P( test.rep(1, sep = ",") ) - val enumByName: P[Ast.expr.EnumByLabel] = P( (NAME) ~ "::" ~ (NAME) ).map { - case(enumName, enumLabel) => Ast.expr.EnumByLabel(enumName, enumLabel) + val enumByName: P[Ast.expr.EnumByLabel] = P("::".!.? ~ NAME.rep(2, "::")).map { + case (first, names: Seq[Ast.identifier]) => + val isAbsolute = first.nonEmpty + val (enumName, enumLabel) = names.takeRight(2) match { + case Seq(a, b) => (a, b) + } + val typePath = names.dropRight(2) + if (typePath.isEmpty) { + Ast.expr.EnumByLabel(enumName, enumLabel, Ast.EmptyTypeId) + } else { + Ast.expr.EnumByLabel(enumName, enumLabel, Ast.typeId(isAbsolute, typePath.map(_.name))) + } } + val byteSizeOfType: P[Ast.expr.ByteSizeOfType] = + P("sizeof" ~ "<" ~ TYPE_NAME ~ ">").map(typeName => Ast.expr.ByteSizeOfType(typeName)) + val bitSizeOfType: P[Ast.expr.BitSizeOfType] = + P("bitsizeof" ~ "<" ~ TYPE_NAME ~ ">").map(typeName => Ast.expr.BitSizeOfType(typeName)) + val topExpr: P[Ast.expr] = P( test ~ End ) + val topExprList: P[Seq[Ast.expr]] = P(testlist1 ~ End) + class ParseException(val src: String, val failure: Parsed.Failure) extends RuntimeException(failure.msg) - def parse(src: String): Ast.expr = { - val r = Expressions.topExpr.parse(src) + def parse(src: String): Ast.expr = realParse(src, topExpr) + def parseList(src: String): Seq[Ast.expr] = realParse(src, topExprList) + + private def realParse[T](src: String, parser: P[T]): T = { + val r = parser.parse(src.trim) r match { - case Parsed.Success(value, index) => value + case Parsed.Success(value, _) => value case f: Parsed.Failure => throw new ParseException(src, f) } diff --git a/shared/src/main/scala/io/kaitai/struct/exprlang/Lexical.scala b/shared/src/main/scala/io/kaitai/struct/exprlang/Lexical.scala index b85d17a18..b04b07e25 100644 --- a/shared/src/main/scala/io/kaitai/struct/exprlang/Lexical.scala +++ b/shared/src/main/scala/io/kaitai/struct/exprlang/Lexical.scala @@ -80,9 +80,9 @@ object Lexical { val bindigit: P0 = P( "0" | "1" | "_" ) val hexdigit: P0 = P( digit | CharIn('a' to 'f', 'A' to 'F') | "_" ) - val floatnumber: P[BigDecimal] = P( pointfloat | exponentfloat ) + val floatnumber: P[BigDecimal] = P( exponentfloat | pointfloat ) val pointfloat: P[BigDecimal] = P( intpart.? ~ fraction | intpart ~ "." ).!.map(BigDecimal(_)) - val exponentfloat: P[BigDecimal] = P( (intpart | pointfloat) ~ exponent ).!.map(BigDecimal(_)) + val exponentfloat: P[BigDecimal] = P( (pointfloat | intpart) ~ exponent ).!.map(BigDecimal(_)) val intpart: P[BigDecimal] = P( digit.rep(1) ).!.map(BigDecimal(_)) val fraction: P0 = P( "." ~ digit.rep(1) ) val exponent: P0 = P( ("e" | "E") ~ ("+" | "-").? ~ digit.rep(1) ) diff --git a/shared/src/main/scala/io/kaitai/struct/format/AttrSpec.scala b/shared/src/main/scala/io/kaitai/struct/format/AttrSpec.scala index c694f729d..9643cd51c 100644 --- a/shared/src/main/scala/io/kaitai/struct/format/AttrSpec.scala +++ b/shared/src/main/scala/io/kaitai/struct/format/AttrSpec.scala @@ -10,28 +10,55 @@ import io.kaitai.struct.exprlang.{Ast, Expressions} import scala.collection.JavaConversions._ -sealed trait RepeatSpec -case class RepeatExpr(expr: Ast.expr) extends RepeatSpec -case class RepeatUntil(expr: Ast.expr) extends RepeatSpec -case object RepeatEos extends RepeatSpec -case object NoRepeat extends RepeatSpec - case class ConditionalSpec(ifExpr: Option[Ast.expr], repeat: RepeatSpec) -trait AttrLikeSpec extends YAMLPath { +trait AttrLikeSpec extends MemberSpec { def dataType: DataType def cond: ConditionalSpec def doc: DocSpec def isArray: Boolean = cond.repeat != NoRepeat - def dataTypeComposite: DataType = { + override def dataTypeComposite: DataType = { if (isArray) { ArrayType(dataType) } else { dataType } } + + override def isNullable: Boolean = { + if (cond.ifExpr.isDefined) { + true + } else { + dataType match { + case st: SwitchType => + st.isNullable + case _ => + false + } + } + } + + def isNullableSwitchRaw: Boolean = { + if (cond.ifExpr.isDefined) { + true + } else { + dataType match { + case st: SwitchType => + st.isNullableSwitchRaw + case _ => + false + } + } + } + + /** + * Determines if this attribute is to be parsed lazily (i.e. on first use), + * or eagerly (during object construction, usually in a `_read` method) + * @return True if this attribute is lazy, false if it's eager + */ + def isLazy: Boolean } case class AttrSpec( @@ -39,8 +66,11 @@ case class AttrSpec( id: Identifier, dataType: DataType, cond: ConditionalSpec = ConditionalSpec(None, NoRepeat), + valid: Option[ValidationSpec] = None, doc: DocSpec = DocSpec.EMPTY -) extends AttrLikeSpec +) extends AttrLikeSpec with MemberSpec { + override def isLazy = false +} case class YamlAttrArgs( size: Option[Ast.expr], @@ -86,6 +116,7 @@ object AttrSpec { "consume", "include", "eos-error", + "valid", "repeat" ) @@ -109,7 +140,7 @@ object AttrSpec { "enum" ) - def fromYaml(src: Any, path: List[String], metaDef: MetaDefaults, idx: Int): AttrSpec = { + def fromYaml(src: Any, path: List[String], metaDef: MetaSpec, idx: Int): AttrSpec = { val srcMap = ParseUtils.asMapStr(src, path) val id = ParseUtils.getOptValueStr(srcMap, "id", path) match { case Some(idStr) => @@ -124,7 +155,7 @@ object AttrSpec { fromYaml(srcMap, path, metaDef, id) } - def fromYaml(srcMap: Map[String, Any], path: List[String], metaDef: MetaDefaults, id: Identifier): AttrSpec = { + def fromYaml(srcMap: Map[String, Any], path: List[String], metaDef: MetaSpec, id: Identifier): AttrSpec = { try { fromYaml2(srcMap, path, metaDef, id) } catch { @@ -133,25 +164,23 @@ object AttrSpec { } } - def fromYaml2(srcMap: Map[String, Any], path: List[String], metaDef: MetaDefaults, id: Identifier): AttrSpec = { + def fromYaml2(srcMap: Map[String, Any], path: List[String], metaDef: MetaSpec, id: Identifier): AttrSpec = { val doc = DocSpec.fromYaml(srcMap, path) - val process = ProcessExpr.fromStr(ParseUtils.getOptValueStr(srcMap, "process", path)) + val process = ProcessExpr.fromStr(ParseUtils.getOptValueStr(srcMap, "process", path), path) // TODO: add proper path propagation val contents = srcMap.get("contents").map(parseContentSpec(_, path ++ List("contents"))) - val size = ParseUtils.getOptValueStr(srcMap, "size", path).map(Expressions.parse) + val size = ParseUtils.getOptValueExpression(srcMap, "size", path) val sizeEos = ParseUtils.getOptValueBool(srcMap, "size-eos", path).getOrElse(false) - val ifExpr = ParseUtils.getOptValueStr(srcMap, "if", path).map(Expressions.parse) + val ifExpr = ParseUtils.getOptValueExpression(srcMap, "if", path) val encoding = ParseUtils.getOptValueStr(srcMap, "encoding", path) - val repeat = ParseUtils.getOptValueStr(srcMap, "repeat", path) - val repeatExpr = ParseUtils.getOptValueStr(srcMap, "repeat-expr", path).map(Expressions.parse) - val repeatUntil = ParseUtils.getOptValueStr(srcMap, "repeat-until", path).map(Expressions.parse) val terminator = ParseUtils.getOptValueInt(srcMap, "terminator", path) val consume = ParseUtils.getOptValueBool(srcMap, "consume", path).getOrElse(true) val include = ParseUtils.getOptValueBool(srcMap, "include", path).getOrElse(false) val eosError = ParseUtils.getOptValueBool(srcMap, "eos-error", path).getOrElse(true) val padRight = ParseUtils.getOptValueInt(srcMap, "pad-right", path) val enum = ParseUtils.getOptValueStr(srcMap, "enum", path) - val parent = ParseUtils.getOptValueStr(srcMap, "parent", path).map(Expressions.parse) + val parent = ParseUtils.getOptValueExpression(srcMap, "parent", path) + val valid = srcMap.get("valid").map(ValidationSpec.fromYaml(_, path ++ List("valid"))) val typObj = srcMap.get("type") @@ -181,20 +210,20 @@ object AttrSpec { } } - val (repeatSpec, legalRepeatKeys) = parseRepeat(repeat, repeatExpr, repeatUntil, path) + val (repeatSpec, legalRepeatKeys) = RepeatSpec.fromYaml(srcMap, path) val legalKeys = LEGAL_KEYS ++ legalRepeatKeys ++ (dataType match { case _: BytesType => LEGAL_KEYS_BYTES case _: StrFromBytesType => LEGAL_KEYS_STR case _: UserType => LEGAL_KEYS_BYTES case EnumType(_, _) => LEGAL_KEYS_ENUM - case SwitchType(on, cases) => LEGAL_KEYS_BYTES + case _: SwitchType => LEGAL_KEYS_BYTES case _ => Set() }) ParseUtils.ensureLegalKeys(srcMap, legalKeys, path) - AttrSpec(path, id, dataType, ConditionalSpec(ifExpr, repeatSpec), doc) + AttrSpec(path, id, dataType, ConditionalSpec(ifExpr, repeatSpec), valid, doc) } def parseContentSpec(c: Any, path: List[String]): Array[Byte] = { @@ -227,23 +256,26 @@ object AttrSpec { private def parseSwitch( switchSpec: Map[String, Any], path: List[String], - metaDef: MetaDefaults, + metaDef: MetaSpec, arg: YamlAttrArgs ): DataType = { - val _on = ParseUtils.getValueStr(switchSpec, "switch-on", path) - val _cases: Map[String, String] = switchSpec.get("cases") match { - case None => Map() - case Some(x) => ParseUtils.asMapStrStr(x, path ++ List("cases")) - } + val on = ParseUtils.getValueExpression(switchSpec, "switch-on", path) + val _cases = ParseUtils.getValueMapStrStr(switchSpec, "cases", path) ParseUtils.ensureLegalKeys(switchSpec, LEGAL_KEYS_SWITCH, path) - val on = Expressions.parse(_on) val cases = _cases.map { case (condition, typeName) => - Expressions.parse(condition) -> DataType.fromYaml( - Some(typeName), path ++ List("cases"), metaDef, + val casePath = path ++ List("cases", condition) + val condType = DataType.fromYaml( + Some(typeName), casePath, metaDef, arg ) + try { + Expressions.parse(condition) -> condType + } catch { + case epe: Expressions.ParseException => + throw YAMLParseException.expression(epe, casePath) + } } // If we have size defined, and we don't have any "else" case already, add @@ -266,42 +298,4 @@ object AttrSpec { SwitchType(on, cases ++ addCases) } - - private def parseRepeat( - repeat: Option[String], - rExpr: Option[Ast.expr], - rUntil: Option[Ast.expr], - path: List[String] - ): (RepeatSpec, Set[String]) = { - repeat match { - case None => - (NoRepeat, Set()) - case Some("until") => - val spec = rUntil match { - case Some(expr) => RepeatUntil(expr) - case None => - throw new YAMLParseException( - "`repeat: until` requires a `repeat-until` expression", - path ++ List("repeat") - ) - } - (spec, Set("repeat-until")) - case Some("expr") => - val spec = rExpr match { - case Some(expr) => RepeatExpr(expr) - case None => - throw new YAMLParseException( - "`repeat: expr` requires a `repeat-expr` expression", - path ++ List("repeat") - ) - } - (spec, Set("repeat-expr")) - case Some("eos") => - (RepeatEos, Set()) - case Some(other) => - throw YAMLParseException.badDictValue( - Set("until", "expr", "eos"), other, path ++ List("repeat") - ) - } - } } diff --git a/shared/src/main/scala/io/kaitai/struct/format/ClassSpec.scala b/shared/src/main/scala/io/kaitai/struct/format/ClassSpec.scala index 4fa168f69..60582238d 100644 --- a/shared/src/main/scala/io/kaitai/struct/format/ClassSpec.scala +++ b/shared/src/main/scala/io/kaitai/struct/format/ClassSpec.scala @@ -1,7 +1,9 @@ package io.kaitai.struct.format import io.kaitai.struct.datatype.DataType -import io.kaitai.struct.datatype.DataType.{KaitaiStructType, UserTypeInstream} +import io.kaitai.struct.datatype.DataType._ + +import scala.collection.mutable /** * Type that we use when we want to refer to a class specification or something @@ -11,11 +13,18 @@ sealed trait ClassSpecLike case object UnknownClassSpec extends ClassSpecLike case object GenericStructClassSpec extends ClassSpecLike +sealed trait Sized +case object DynamicSized extends Sized +case object NotCalculatedSized extends Sized +case object StartedCalculationSized extends Sized +case class FixedSized(n: Int) extends Sized + case class ClassSpec( path: List[String], isTopLevel: Boolean, - meta: Option[MetaSpec], + meta: MetaSpec, doc: DocSpec, + params: List[ParamDefSpec], seq: List[AttrSpec], types: Map[String, ClassSpec], instances: Map[InstanceIdentifier, InstanceSpec], @@ -42,14 +51,37 @@ case class ClassSpec( */ var upClass: Option[ClassSpec] = None - def parentTypeName: List[String] = parentClass match { - case UnknownClassSpec | GenericStructClassSpec => List("kaitai_struct") - case t: ClassSpec => t.name - } + var seqSize: Sized = NotCalculatedSized def parentType: DataType = parentClass match { - case UnknownClassSpec | GenericStructClassSpec => KaitaiStructType - case t: ClassSpec => UserTypeInstream(t.name, None) + case UnknownClassSpec | GenericStructClassSpec => CalcKaitaiStructType + case t: ClassSpec => CalcUserType(t.name, None) + } + + /** + * Recursively traverses tree of types starting from this type, calling + * certain function for every type, starting from this one. + */ + def forEachRec(proc: (ClassSpec) => Unit): Unit = { + proc.apply(this) + types.foreach { case (_, typeSpec) => + typeSpec.forEachRec(proc) + } + } + + override def equals(obj: Any): Boolean = obj match { + case other: ClassSpec => + path == other.path && + isTopLevel == other.isTopLevel && + meta == other.meta && + doc == other.doc && + params == other.params && + seq == other.seq && + types == other.types && + instances == other.instances && + enums == other.enums && + name == other.name + case _ => false } } @@ -58,31 +90,37 @@ object ClassSpec { "meta", "doc", "doc-ref", + "params", "seq", "types", "instances", "enums" ) - def fromYaml(src: Any, path: List[String], metaDef: MetaDefaults): ClassSpec = { + def fromYaml(src: Any, path: List[String], metaDef: MetaSpec): ClassSpec = { val srcMap = ParseUtils.asMapStr(src, path) ParseUtils.ensureLegalKeys(srcMap, LEGAL_KEYS, path) - val meta = srcMap.get("meta").map(MetaSpec.fromYaml(_, path ++ List("meta"))) - val curMetaDef = metaDef.updateWith(meta) + val metaPath = path ++ List("meta") + val explicitMeta = srcMap.get("meta").map(MetaSpec.fromYaml(_, metaPath)).getOrElse(MetaSpec.emptyWithPath(metaPath)) + val meta = explicitMeta.fillInDefaults(metaDef) val doc = DocSpec.fromYaml(srcMap, path) + val params: List[ParamDefSpec] = srcMap.get("params") match { + case Some(value) => paramDefFromYaml(value, path ++ List("params")) + case None => List() + } val seq: List[AttrSpec] = srcMap.get("seq") match { - case Some(value) => seqFromYaml(value, path ++ List("seq"), curMetaDef) + case Some(value) => seqFromYaml(value, path ++ List("seq"), meta) case None => List() } val types: Map[String, ClassSpec] = srcMap.get("types") match { - case Some(value) => typesFromYaml(value, path ++ List("types"), curMetaDef) + case Some(value) => typesFromYaml(value, path ++ List("types"), meta) case None => Map() } val instances: Map[InstanceIdentifier, InstanceSpec] = srcMap.get("instances") match { - case Some(value) => instancesFromYaml(value, path ++ List("instances"), curMetaDef) + case Some(value) => instancesFromYaml(value, path ++ List("instances"), meta) case None => Map() } val enums: Map[String, EnumSpec] = srcMap.get("enums") match { @@ -90,13 +128,17 @@ object ClassSpec { case None => Map() } - val cs = ClassSpec(path, path.isEmpty, meta, doc, seq, types, instances, enums) + checkDupSeqInstIds(seq, instances) + + val cs = ClassSpec( + path, path.isEmpty, + meta, doc, + params, seq, types, instances, enums + ) // If that's a top-level class, set its name from meta/id if (path.isEmpty) { - if (meta.isEmpty) - throw new YAMLParseException("no `meta` encountered in top-level class spec", path) - meta.get.id match { + explicitMeta.id match { case None => throw new YAMLParseException("no `meta/id` encountered in top-level class spec", path ++ List("meta", "id")) case Some(id) => @@ -107,18 +149,68 @@ object ClassSpec { cs } - def seqFromYaml(src: Any, path: List[String], metaDef: MetaDefaults): List[AttrSpec] = { + def paramDefFromYaml(src: Any, path: List[String]): List[ParamDefSpec] = { src match { case srcList: List[Any] => - srcList.zipWithIndex.map { case (attrSrc, idx) => + val params = srcList.zipWithIndex.map { case (attrSrc, idx) => + ParamDefSpec.fromYaml(attrSrc, path ++ List(idx.toString), idx) + } + // FIXME: checkDupSeqIds(params) + params + case unknown => + throw new YAMLParseException(s"expected array, found $unknown", path) + } + } + + def seqFromYaml(src: Any, path: List[String], metaDef: MetaSpec): List[AttrSpec] = { + src match { + case srcList: List[Any] => + val seq = srcList.zipWithIndex.map { case (attrSrc, idx) => AttrSpec.fromYaml(attrSrc, path ++ List(idx.toString), metaDef, idx) } + checkDupSeqIds(seq) + seq case unknown => throw new YAMLParseException(s"expected array, found $unknown", path) } } - def typesFromYaml(src: Any, path: List[String], metaDef: MetaDefaults): Map[String, ClassSpec] = { + def checkDupSeqIds(seq: List[AttrSpec]): Unit = { + val attrIds = mutable.Map[String, AttrSpec]() + seq.foreach { (attr) => + attr.id match { + case NamedIdentifier(id) => + checkDupId(attrIds.get(id), id, attr) + attrIds.put(id, attr) + case _ => // do nothing with non-named IDs + } + } + } + + def checkDupSeqInstIds(seq: List[AttrSpec], instances: Map[InstanceIdentifier, InstanceSpec]): Unit = { + val attrIds: Map[String, AttrSpec] = seq.flatMap((attr) => attr.id match { + case NamedIdentifier(id) => Some(id -> attr) + case _ => None + }).toMap + + instances.foreach { case (id, instSpec) => + checkDupId(attrIds.get(id.name), id.name, instSpec) + } + } + + private def checkDupId(prevAttrOpt: Option[AttrSpec], id: String, nowAttr: YAMLPath) { + prevAttrOpt match { + case Some(prevAttr) => + throw new YAMLParseException( + s"duplicate attribute ID '$id', previously defined at /${prevAttr.pathStr}", + nowAttr.path + ) + case None => + // no dups, ok + } + } + + def typesFromYaml(src: Any, path: List[String], metaDef: MetaSpec): Map[String, ClassSpec] = { val srcMap = ParseUtils.asMapStr(src, path) srcMap.map { case (typeName, body) => Identifier.checkIdentifierSource(typeName, "type", path ++ List(typeName)) @@ -126,7 +218,7 @@ object ClassSpec { } } - def instancesFromYaml(src: Any, path: List[String], metaDef: MetaDefaults): Map[InstanceIdentifier, InstanceSpec] = { + def instancesFromYaml(src: Any, path: List[String], metaDef: MetaSpec): Map[InstanceIdentifier, InstanceSpec] = { val srcMap = ParseUtils.asMap(src, path) srcMap.map { case (key, body) => val instName = ParseUtils.asStr(key, path) @@ -145,14 +237,15 @@ object ClassSpec { } } - def fromYaml(src: Any): ClassSpec = fromYaml(src, List(), MetaDefaults(None, None)) + def fromYaml(src: Any): ClassSpec = fromYaml(src, List(), MetaSpec.OPAQUE) def opaquePlaceholder(typeName: List[String]): ClassSpec = { val placeholder = ClassSpec( List(), true, - meta = Some(MetaSpec.OPAQUE), + meta = MetaSpec.OPAQUE, doc = DocSpec.EMPTY, + params = List(), seq = List(), types = Map(), instances = Map(), diff --git a/shared/src/main/scala/io/kaitai/struct/format/ClassSpecs.scala b/shared/src/main/scala/io/kaitai/struct/format/ClassSpecs.scala index 24837174d..664ce5297 100644 --- a/shared/src/main/scala/io/kaitai/struct/format/ClassSpecs.scala +++ b/shared/src/main/scala/io/kaitai/struct/format/ClassSpecs.scala @@ -1,5 +1,7 @@ package io.kaitai.struct.format +import io.kaitai.struct.precompile.ErrorInInput + import scala.collection.mutable import scala.concurrent.Future @@ -12,6 +14,29 @@ import scala.concurrent.Future abstract class ClassSpecs(val firstSpec: ClassSpec) extends mutable.HashMap[String, ClassSpec] { this(firstSpec.name.head) = firstSpec + /** + * Calls certain function on all [[ClassSpec]] elements stored in this ClassSpecs, + * and all subtypes stored in these elements, recursively. + */ + def forEachRec(proc: (ClassSpec) => Unit): Unit = + forEachTopLevel((_, typeSpec) => typeSpec.forEachRec(proc)) + + /** + * Calls certain function on all top-level [[ClassSpec]] elements stored in this + * ClassSpecs. + */ + def forEachTopLevel(proc: (String, ClassSpec) => Unit): Unit = { + foreach { case (specName, typeSpec) => + try { + proc(specName, typeSpec) + } catch { + case ErrorInInput(err, path, None) => + // Try to emit more specific error, with a reference to current file + throw ErrorInInput(err, path, Some(specName)) + } + } + } + def importRelative(name: String, path: List[String], inFile: Option[String]): Future[Option[ClassSpec]] def importAbsolute(name: String, path: List[String], inFile: Option[String]): Future[Option[ClassSpec]] } diff --git a/shared/src/main/scala/io/kaitai/struct/format/DocSpec.scala b/shared/src/main/scala/io/kaitai/struct/format/DocSpec.scala index e4dea0ba3..86ec070be 100644 --- a/shared/src/main/scala/io/kaitai/struct/format/DocSpec.scala +++ b/shared/src/main/scala/io/kaitai/struct/format/DocSpec.scala @@ -13,38 +13,38 @@ case class UrlRef(url: String, text: String) extends RefSpec { def toAhref: String = "" + text + "" } -case object NoRef extends RefSpec case class DocSpec( summary: Option[String], - ref: RefSpec + ref: List[RefSpec] ) { - def isEmpty: Boolean = summary.isEmpty && ref == NoRef + def isEmpty: Boolean = summary.isEmpty && ref.isEmpty } object DocSpec { - val EMPTY = DocSpec(None, NoRef) + val EMPTY = DocSpec(None, List()) def fromYaml(srcMap: Map[String, Any], path: List[String]): DocSpec = { val doc = ParseUtils.getOptValueStr(srcMap, "doc", path) - val docRefOpt = ParseUtils.getOptValueStr(srcMap, "doc-ref", path) - - val refSpec: RefSpec = docRefOpt.map { (docRef) => - if (docRef.startsWith("http://") || docRef.startsWith("https://")) { - val splitPoint = docRef.indexOf(' ') - if (splitPoint < 0) { - UrlRef(docRef, "Source") - } else { - val url = docRef.substring(0, splitPoint).trim - val text = docRef.substring(splitPoint + 1).trim - UrlRef(url, text) - } - } else { - TextRef(docRef) - } - }.getOrElse(NoRef) + val docRefs = ParseUtils.getListStr(srcMap, "doc-ref", path) + val refSpec = docRefs.map(docRef => parseSingleRefSpec(docRef)) DocSpec(doc, refSpec) } + + def parseSingleRefSpec(docRef: String): RefSpec = { + if (docRef.startsWith("http://") || docRef.startsWith("https://")) { + val splitPoint = docRef.indexOf(' ') + if (splitPoint < 0) { + UrlRef(docRef, "Source") + } else { + val url = docRef.substring(0, splitPoint).trim + val text = docRef.substring(splitPoint + 1).trim + UrlRef(url, text) + } + } else { + TextRef(docRef) + } + } } diff --git a/shared/src/main/scala/io/kaitai/struct/format/EnumSpec.scala b/shared/src/main/scala/io/kaitai/struct/format/EnumSpec.scala index f7db4f515..2794e0b85 100644 --- a/shared/src/main/scala/io/kaitai/struct/format/EnumSpec.scala +++ b/shared/src/main/scala/io/kaitai/struct/format/EnumSpec.scala @@ -1,25 +1,29 @@ package io.kaitai.struct.format -case class EnumSpec(map: Map[Long, String]) { +case class EnumSpec(map: Map[Long, EnumValueSpec]) { var name = List[String]() + /** + * @return Absolute name of enum as string, components separated by + * double colon operator `::` + */ + def nameAsStr = name.mkString("::") + /** * Stabilize order of generated enums by sorting it by integer ID - it * both looks nicer and doesn't screw diffs in generated code. */ - lazy val sortedSeq: Seq[(Long, String)] = map.toSeq.sortBy(_._1) + lazy val sortedSeq: Seq[(Long, EnumValueSpec)] = map.toSeq.sortBy(_._1) } object EnumSpec { def fromYaml(src: Any, path: List[String]): EnumSpec = { val srcMap = ParseUtils.asMap(src, path) - EnumSpec(srcMap.map { case (id, name) => + EnumSpec(srcMap.map { case (id, desc) => val idLong = ParseUtils.asLong(id, path) - val symbName = ParseUtils.asStr(name, path ++ List(idLong.toString)) - - Identifier.checkIdentifierSource(symbName, "enum member", path ++ List(idLong.toString)) + val value = EnumValueSpec.fromYaml(desc, path ++ List(idLong.toString)) - idLong -> symbName + idLong -> value }) } } diff --git a/shared/src/main/scala/io/kaitai/struct/format/EnumValueSpec.scala b/shared/src/main/scala/io/kaitai/struct/format/EnumValueSpec.scala new file mode 100644 index 000000000..905e968a6 --- /dev/null +++ b/shared/src/main/scala/io/kaitai/struct/format/EnumValueSpec.scala @@ -0,0 +1,40 @@ +package io.kaitai.struct.format + +case class EnumValueSpec(name: String, doc: DocSpec) + +object EnumValueSpec { + def fromYaml(src: Any, path: List[String]): EnumValueSpec = { + src match { + case name: String => + fromSimpleName(name, path) + case x: Boolean => + fromSimpleName(x.toString, path) + case srcMap: Map[Any, Any] => + fromMap(ParseUtils.anyMapToStrMap(srcMap, path), path) + case _ => + throw YAMLParseException.badType("string or map", src, path) + } + } + + def fromSimpleName(name: String, path: List[String]): EnumValueSpec = { + Identifier.checkIdentifierSource(name, "enum member", path) + EnumValueSpec(name, DocSpec.EMPTY) + } + + val LEGAL_KEYS = Set( + "id", + "doc", + "doc-ref" + ) + + def fromMap(srcMap: Map[String, Any], path: List[String]): EnumValueSpec = { + ParseUtils.ensureLegalKeys(srcMap, LEGAL_KEYS, path, Some("enum value spec")) + + val name = ParseUtils.getValueStr(srcMap, "id", path) + Identifier.checkIdentifierSource(name, "enum value spec id", path) + + val doc = DocSpec.fromYaml(srcMap, path) + + EnumValueSpec(name, doc) + } +} diff --git a/shared/src/main/scala/io/kaitai/struct/format/Identifier.scala b/shared/src/main/scala/io/kaitai/struct/format/Identifier.scala index f907de756..84146005e 100644 --- a/shared/src/main/scala/io/kaitai/struct/format/Identifier.scala +++ b/shared/src/main/scala/io/kaitai/struct/format/Identifier.scala @@ -1,15 +1,31 @@ package io.kaitai.struct.format +import io.kaitai.struct.exprlang.Ast + /** * Common abstract container for all identifiers that Kaitai Struct deals with. */ -abstract class Identifier +abstract class Identifier { + /** + * @return Human-readable name of identifier, to be used exclusively for ksc + * error messaging purposes. + */ + def humanReadable: String + + /** + * @return Identifier ready to be used in KS expressions. + */ + def toAstIdentifier: Ast.identifier = Ast.identifier(humanReadable) +} /** * Identifier generated automatically for seq attributes which lack true string "id" field. * @param idx unique number to identify attribute with */ -case class NumberedIdentifier(idx: Int) extends Identifier +case class NumberedIdentifier(idx: Int) extends Identifier { + import NumberedIdentifier._ + override def humanReadable: String = s"${TEMPLATE}_$idx" +} object NumberedIdentifier { val TEMPLATE = "unnamed" @@ -21,9 +37,13 @@ object NumberedIdentifier { */ case class NamedIdentifier(name: String) extends Identifier { Identifier.checkIdentifier(name) + + override def humanReadable: String = name } -case class InvalidIdentifier(id: String) extends RuntimeException +case class InvalidIdentifier(id: String) extends RuntimeException( + s"invalid ID: '$id', expected /${Identifier.ReIdentifier.toString}/" +) object Identifier { val ReIdentifier = "^[a-z][a-z0-9_]*$".r @@ -60,19 +80,31 @@ object Identifier { val IO = "_io" val ITERATOR = "_" val ITERATOR2 = "_buf" - val ITERATOR_I = "_i" + val INDEX = "_index" + val SWITCH_ON = "_on" + val IS_LE = "_is_le" + val SIZEOF = "_sizeof" } -case class RawIdentifier(innerId: Identifier) extends Identifier +case class RawIdentifier(innerId: Identifier) extends Identifier { + override def humanReadable: String = s"raw(${innerId.humanReadable})" +} -case class IoStorageIdentifier(innerId: Identifier) extends Identifier +case class IoStorageIdentifier(innerId: Identifier) extends Identifier { + override def humanReadable: String = s"io(${innerId.humanReadable})" +} case class InstanceIdentifier(name: String) extends Identifier { Identifier.checkIdentifier(name) + + override def humanReadable: String = name } -case class SpecialIdentifier(name: String) extends Identifier +case class SpecialIdentifier(name: String) extends Identifier { + override def humanReadable: String = name +} object RootIdentifier extends SpecialIdentifier(Identifier.ROOT) object ParentIdentifier extends SpecialIdentifier(Identifier.PARENT) object IoIdentifier extends SpecialIdentifier(Identifier.IO) +object EndianIdentifier extends SpecialIdentifier(Identifier.IS_LE) diff --git a/shared/src/main/scala/io/kaitai/struct/format/InstanceSpec.scala b/shared/src/main/scala/io/kaitai/struct/format/InstanceSpec.scala index 024313427..4da5050c8 100644 --- a/shared/src/main/scala/io/kaitai/struct/format/InstanceSpec.scala +++ b/shared/src/main/scala/io/kaitai/struct/format/InstanceSpec.scala @@ -3,8 +3,9 @@ package io.kaitai.struct.format import io.kaitai.struct.datatype.DataType import io.kaitai.struct.exprlang.{Ast, Expressions} -sealed abstract class InstanceSpec(val doc: DocSpec) { +sealed abstract class InstanceSpec(val doc: DocSpec) extends YAMLPath { def dataTypeComposite: DataType + def isNullable: Boolean } case class ValueInstanceSpec( path: List[String], @@ -12,17 +13,21 @@ case class ValueInstanceSpec( value: Ast.expr, ifExpr: Option[Ast.expr], var dataType: Option[DataType] -) extends InstanceSpec(_doc) with YAMLPath { +) extends InstanceSpec(_doc) { override def dataTypeComposite = dataType.get + override def isNullable: Boolean = ifExpr.isDefined } case class ParseInstanceSpec( + id: InstanceIdentifier, path: List[String], private val _doc: DocSpec, dataType: DataType, cond: ConditionalSpec, pos: Option[Ast.expr], io: Option[Ast.expr] -) extends InstanceSpec(_doc) with AttrLikeSpec with YAMLPath +) extends InstanceSpec(_doc) with AttrLikeSpec { + override def isLazy = true +} object InstanceSpec { val LEGAL_KEYS_VALUE_INST = Set( @@ -33,10 +38,10 @@ object InstanceSpec { "if" ) - def fromYaml(src: Any, path: List[String], metaDef: MetaDefaults, id: InstanceIdentifier): InstanceSpec = { + def fromYaml(src: Any, path: List[String], metaDef: MetaSpec, id: InstanceIdentifier): InstanceSpec = { val srcMap = ParseUtils.asMapStr(src, path) - ParseUtils.getOptValueStr(srcMap, "value", path).map(Expressions.parse) match { + ParseUtils.getOptValueExpression(srcMap, "value", path) match { case Some(value) => // value instance ParseUtils.ensureLegalKeys(srcMap, LEGAL_KEYS_VALUE_INST, path, Some("value instance")) @@ -49,7 +54,7 @@ object InstanceSpec { Ast.expr.EnumById(Ast.identifier(enumName), value) } - val ifExpr = ParseUtils.getOptValueStr(srcMap, "if", path).map(Expressions.parse) + val ifExpr = ParseUtils.getOptValueExpression(srcMap, "if", path) ValueInstanceSpec( path, @@ -60,12 +65,12 @@ object InstanceSpec { ) case None => // normal positional instance - val pos = ParseUtils.getOptValueStr(srcMap, "pos", path).map(Expressions.parse) - val io = ParseUtils.getOptValueStr(srcMap, "io", path).map(Expressions.parse) + val pos = ParseUtils.getOptValueExpression(srcMap, "pos", path) + val io = ParseUtils.getOptValueExpression(srcMap, "io", path) val fakeAttrMap = srcMap.filterKeys((key) => key != "pos" && key != "io") val a = AttrSpec.fromYaml(fakeAttrMap, path, metaDef, id) - ParseInstanceSpec(path, a.doc, a.dataType, a.cond, pos, io) + ParseInstanceSpec(id, path, a.doc, a.dataType, a.cond, pos, io) } } } diff --git a/shared/src/main/scala/io/kaitai/struct/format/KSVersion.scala b/shared/src/main/scala/io/kaitai/struct/format/KSVersion.scala index b41836555..766bd891e 100644 --- a/shared/src/main/scala/io/kaitai/struct/format/KSVersion.scala +++ b/shared/src/main/scala/io/kaitai/struct/format/KSVersion.scala @@ -56,6 +56,14 @@ case class KSVersion(nums: List[Int]) extends Ordered[KSVersion] { } object KSVersion { + /** + * This is abomination, the sole purpose of which is to get away from atrocious + * SBT "we can generate managed source files (=Version.scala) only in platform- + * dependent projects". As "shared" is not a such project, we can't just directly + * generate a file that will be used by both projects. Probably something can be + * done about it, but I've already spent like 4-5 hours on it and I'd rather spend + * more on something more productive. + */ private var _current: Option[KSVersion] = None def current_=(str: String) { @@ -65,7 +73,7 @@ object KSVersion { def current: KSVersion = _current.get def fromStr(str: String): KSVersion = - KSVersion(str.replace("-SNAPSHOT", "").split('.').map(_.toInt).toList) + KSVersion(str.replaceAll("-SNAPSHOT.*$", "").split('.').map(_.toInt).toList) /** * Hardcoded minimal version of runtime API that this particular @@ -74,5 +82,5 @@ object KSVersion { * language supports it) when trying to use generated file together * with this older runtime API. */ - val minimalRuntime: KSVersion = KSVersion.fromStr("0.7") + val minimalRuntime: KSVersion = KSVersion.fromStr("0.9") } diff --git a/shared/src/main/scala/io/kaitai/struct/format/MemberSpec.scala b/shared/src/main/scala/io/kaitai/struct/format/MemberSpec.scala new file mode 100644 index 000000000..c8ed8ce3e --- /dev/null +++ b/shared/src/main/scala/io/kaitai/struct/format/MemberSpec.scala @@ -0,0 +1,31 @@ +package io.kaitai.struct.format + +import io.kaitai.struct.datatype.DataType + +/** + * Base trait for everything that would be compiled to be members of the class, + * i.e. sequence attributes, parse instances, value instances, parameters. + */ +trait MemberSpec extends YAMLPath { + def id: Identifier + def dataType: DataType + def doc: DocSpec + + def dataTypeComposite = dataType + + /** + * Determines if this attribute can be "null" in some circumstances or not. + * In some target languages, it would affect data types used, init and + * cleanup procedures. + * @return True if this attribute can be "null", false if it's never "null" + */ + def isNullable: Boolean + + /** + * Determines if this attribute can be "null" in some circumstances or not. + * This version is for languages like C++, which make a special exception + * for raw byte arrays placement for switch statements. + * @return True if this attribute can be "null", false if it's never "null" + */ + def isNullableSwitchRaw: Boolean +} diff --git a/shared/src/main/scala/io/kaitai/struct/format/MetaDefaults.scala b/shared/src/main/scala/io/kaitai/struct/format/MetaDefaults.scala deleted file mode 100644 index 928f578cd..000000000 --- a/shared/src/main/scala/io/kaitai/struct/format/MetaDefaults.scala +++ /dev/null @@ -1,19 +0,0 @@ -package io.kaitai.struct.format - -import io.kaitai.struct.datatype.Endianness - -case class MetaDefaults( - endian: Option[Endianness], - encoding: Option[String] -) { - def updateWith(metaOpt: Option[MetaSpec]): MetaDefaults = { - metaOpt match { - case None => this - case Some(meta) => - MetaDefaults( - meta.endian.orElse(this.endian), - meta.encoding.orElse(this.encoding) - ) - } - } -} diff --git a/shared/src/main/scala/io/kaitai/struct/format/MetaSpec.scala b/shared/src/main/scala/io/kaitai/struct/format/MetaSpec.scala index abb8a6690..9b6ea51e8 100644 --- a/shared/src/main/scala/io/kaitai/struct/format/MetaSpec.scala +++ b/shared/src/main/scala/io/kaitai/struct/format/MetaSpec.scala @@ -1,6 +1,6 @@ package io.kaitai.struct.format -import io.kaitai.struct.datatype.Endianness +import io.kaitai.struct.datatype.{CalcEndian, Endianness, InheritedEndian} case class MetaSpec( path: List[String], @@ -11,9 +11,37 @@ case class MetaSpec( forceDebug: Boolean, opaqueTypes: Option[Boolean], imports: List[String] -) extends YAMLPath +) extends YAMLPath { + def fillInDefaults(defSpec: MetaSpec): MetaSpec = { + fillInEncoding(defSpec.encoding). + fillInEndian(defSpec.endian) + } + + private + def fillInEncoding(defEncoding: Option[String]): MetaSpec = { + (defEncoding, encoding) match { + case (None, _) => this + case (_, Some(_)) => this + case (Some(_), None) => + this.copy(encoding = defEncoding) + } + } + + def fillInEndian(defEndian: Option[Endianness]): MetaSpec = { + (defEndian, endian) match { + case (None, _) => this + case (_, Some(_)) => this + case (Some(_: CalcEndian), None) => + this.copy(endian = Some(InheritedEndian)) + case (Some(_), None) => + this.copy(endian = defEndian) + } + } +} object MetaSpec { + def emptyWithPath(path: List[String]) = OPAQUE.copy(isOpaque = false, path = path) + val OPAQUE = MetaSpec( path = List(), isOpaque = true, @@ -36,6 +64,8 @@ object MetaSpec { "ks-opaque-types", "license", "file-extension", + "xref", + "tags", "application" ) @@ -48,6 +78,8 @@ object MetaSpec { throw YAMLParseException.incompatibleVersion(ver, KSVersion.current, path) } + val endian: Option[Endianness] = Endianness.fromYaml(srcMap.get("endian"), path) + ParseUtils.ensureLegalKeys(srcMap, LEGAL_KEYS, path) val id = ParseUtils.getOptValueStr(srcMap, "id", path) @@ -55,10 +87,6 @@ object MetaSpec { Identifier.checkIdentifierSource(idStr, "meta", path ++ List("id")) ) - val endian: Option[Endianness] = Endianness.defaultFromString( - ParseUtils.getOptValueStr(srcMap, "endian", path), - path - ) val encoding = ParseUtils.getOptValueStr(srcMap, "encoding", path) val forceDebug = ParseUtils.getOptValueBool(srcMap, "ks-debug", path).getOrElse(false) diff --git a/shared/src/main/scala/io/kaitai/struct/format/ParamDefSpec.scala b/shared/src/main/scala/io/kaitai/struct/format/ParamDefSpec.scala new file mode 100644 index 000000000..5d25eaf62 --- /dev/null +++ b/shared/src/main/scala/io/kaitai/struct/format/ParamDefSpec.scala @@ -0,0 +1,41 @@ +package io.kaitai.struct.format + +import io.kaitai.struct.datatype.DataType + +case class ParamDefSpec( + path: List[String], + id: Identifier, + dataType: DataType, + doc: DocSpec = DocSpec.EMPTY +) extends MemberSpec { + override def isNullable: Boolean = false + override def isNullableSwitchRaw: Boolean = false +} + +object ParamDefSpec { + def fromYaml(src: Any, path: List[String], idx: Int): ParamDefSpec = { + val srcMap = ParseUtils.asMapStr(src, path) + val id = ParseUtils.getValueIdentifier(srcMap, idx, "parameter", path) + fromYaml(srcMap, path, id) + } + + val LEGAL_KEYS = Set( + "id", + "type", + "enum", + "doc", + "doc-ref" + ) + + def fromYaml(srcMap: Map[String, Any], path: List[String], id: Identifier): ParamDefSpec = { + val doc = DocSpec.fromYaml(srcMap, path) + val typeStr = ParseUtils.getOptValueStr(srcMap, "type", path) + val enumRef = ParseUtils.getOptValueStr(srcMap, "enum", path) + + val dataType = DataType.pureFromString(typeStr, enumRef, path) + + ParseUtils.ensureLegalKeys(srcMap, LEGAL_KEYS, path, Some("parameter definition")) + + ParamDefSpec(path, id, dataType, doc) + } +} diff --git a/shared/src/main/scala/io/kaitai/struct/format/ParseUtils.scala b/shared/src/main/scala/io/kaitai/struct/format/ParseUtils.scala index 09e589827..dd70279ba 100644 --- a/shared/src/main/scala/io/kaitai/struct/format/ParseUtils.scala +++ b/shared/src/main/scala/io/kaitai/struct/format/ParseUtils.scala @@ -1,6 +1,7 @@ package io.kaitai.struct.format import io.kaitai.struct.Utils +import io.kaitai.struct.exprlang.{Ast, Expressions} object ParseUtils { def ensureLegalKeys(src: Map[String, Any], legalKeys: Set[String], path: List[String], where: Option[String] = None) = { @@ -21,10 +22,19 @@ object ParseUtils { def getValueStr(src: Map[String, Any], field: String, path: List[String]): String = { src.get(field) match { - case Some(value: String) => - value - case unknown => - throw YAMLParseException.badType("string", unknown, path ++ List(field)) + case Some(value) => + asStr(value, path ++ List(field)) + case None => + throw YAMLParseException.noKey(path ++ List(field)) + } + } + + def getValueMapStrStr(src: Map[String, Any], field: String, path: List[String]): Map[String, String] = { + src.get(field) match { + case Some(value) => + asMapStrStr(value, path ++ List(field)) + case None => + throw YAMLParseException.noKey(path ++ List(field)) } } @@ -61,6 +71,37 @@ object ParseUtils { } } + def getValueIdentifier(src: Map[String, Any], idx: Int, entityName: String, path: List[String]): Identifier = { + getOptValueStr(src, "id", path) match { + case Some(idStr) => + try { + NamedIdentifier(idStr) + } catch { + case _: InvalidIdentifier => + throw YAMLParseException.invalidId(idStr, entityName, path ++ List("id")) + } + case None => NumberedIdentifier(idx) + } + } + + def getValueExpression(src: Map[String, Any], field: String, path: List[String]): Ast.expr = { + try { + Expressions.parse(getValueStr(src, field, path)) + } catch { + case epe: Expressions.ParseException => + throw YAMLParseException.expression(epe, path) + } + } + + def getOptValueExpression(src: Map[String, Any], field: String, path: List[String]): Option[Ast.expr] = { + try { + getOptValueStr(src, field, path).map(Expressions.parse) + } catch { + case epe: Expressions.ParseException => + throw YAMLParseException.expression(epe, path) + } + } + /** * Gets a list of T-typed values from a given YAML map's key "field", * reporting errors accurately and ensuring type safety. @@ -89,6 +130,8 @@ object ParseUtils { srcList.zipWithIndex.map { case (element, idx) => convertFunc(element, pathField ++ List(idx.toString)) } + case Some(singleObject: T) => + List(singleObject) case None => List() case unknown => @@ -113,6 +156,8 @@ object ParseUtils { str case n: Int => n.toString + case n: Long => + n.toString case n: Double => n.toString case n: Boolean => diff --git a/shared/src/main/scala/io/kaitai/struct/format/ProcessExpr.scala b/shared/src/main/scala/io/kaitai/struct/format/ProcessExpr.scala index 3f56f2b50..9764f8731 100644 --- a/shared/src/main/scala/io/kaitai/struct/format/ProcessExpr.scala +++ b/shared/src/main/scala/io/kaitai/struct/format/ProcessExpr.scala @@ -1,45 +1,44 @@ package io.kaitai.struct.format -import io.kaitai.struct.exprlang.{Expressions, Ast} +import io.kaitai.struct.exprlang.{Ast, Expressions} -trait ProcessExpr { - def outputType: String -} +sealed trait ProcessExpr -case object ProcessZlib extends ProcessExpr { - override def outputType: String = null -} -case object ProcessHexStrToInt extends ProcessExpr { - override def outputType: String = "u4" -} -case class ProcessXor(key: Ast.expr) extends ProcessExpr { - override def outputType: String = null -} -case class ProcessRotate(left: Boolean, key: Ast.expr) extends ProcessExpr { - override def outputType: String = null -} +case object ProcessZlib extends ProcessExpr +case class ProcessXor(key: Ast.expr) extends ProcessExpr +case class ProcessRotate(left: Boolean, key: Ast.expr) extends ProcessExpr +case class ProcessCustom(name: List[String], args: Seq[Ast.expr]) extends ProcessExpr object ProcessExpr { private val ReXor = "^xor\\(\\s*(.*?)\\s*\\)$".r private val ReRotate = "^ro(l|r)\\(\\s*(.*?)\\s*\\)$".r + private val ReCustom = "^([a-z][a-z0-9_.]*)\\(\\s*(.*?)\\s*\\)$".r + private val ReCustomNoArg = "^([a-z][a-z0-9_.]*)$".r - def fromStr(s: Option[String]): Option[ProcessExpr] = { + def fromStr(s: Option[String], path: List[String]): Option[ProcessExpr] = { s match { case None => None case Some(op) => - Some(op match { - case "zlib" => - ProcessZlib - case "hexstr_to_int" => - ProcessHexStrToInt - case ReXor(arg) => - ProcessXor(Expressions.parse(arg)) - case ReRotate(dir, arg) => - ProcessRotate(dir == "l", Expressions.parse(arg)) - case _ => - throw new RuntimeException(s"Invalid process: '$s'") - }) + try { + Some(op match { + case "zlib" => + ProcessZlib + case ReXor(arg) => + ProcessXor(Expressions.parse(arg)) + case ReRotate(dir, arg) => + ProcessRotate(dir == "l", Expressions.parse(arg)) + case ReCustom(name, args) => + ProcessCustom(name.split('.').toList, Expressions.parseList(args)) + case ReCustomNoArg(name) => + ProcessCustom(name.split('.').toList, Seq()) + case _ => + throw YAMLParseException.badProcess(op, path) + }) + } catch { + case epe: Expressions.ParseException => + throw YAMLParseException.expression(epe, path) + } } } } diff --git a/shared/src/main/scala/io/kaitai/struct/format/RepeatSpec.scala b/shared/src/main/scala/io/kaitai/struct/format/RepeatSpec.scala new file mode 100644 index 000000000..01187c090 --- /dev/null +++ b/shared/src/main/scala/io/kaitai/struct/format/RepeatSpec.scala @@ -0,0 +1,51 @@ +package io.kaitai.struct.format + +import io.kaitai.struct.exprlang.Ast + +sealed trait RepeatSpec +case class RepeatExpr(expr: Ast.expr) extends RepeatSpec +case class RepeatUntil(expr: Ast.expr) extends RepeatSpec +case object RepeatEos extends RepeatSpec +case object NoRepeat extends RepeatSpec + +object RepeatSpec { + def fromYaml( + srcMap: Map[String, Any], + path: List[String] + ): (RepeatSpec, Set[String]) = { + val repeat = ParseUtils.getOptValueStr(srcMap, "repeat", path) + val repeatExpr = ParseUtils.getOptValueExpression(srcMap, "repeat-expr", path) + val repeatUntil = ParseUtils.getOptValueExpression(srcMap, "repeat-until", path) + + repeat match { + case None => + (NoRepeat, Set()) + case Some("until") => + val spec = repeatUntil match { + case Some(expr) => RepeatUntil(expr) + case None => + throw new YAMLParseException( + "`repeat: until` requires a `repeat-until` expression", + path ++ List("repeat") + ) + } + (spec, Set("repeat-until")) + case Some("expr") => + val spec = repeatExpr match { + case Some(expr) => RepeatExpr(expr) + case None => + throw new YAMLParseException( + "`repeat: expr` requires a `repeat-expr` expression", + path ++ List("repeat") + ) + } + (spec, Set("repeat-expr")) + case Some("eos") => + (RepeatEos, Set()) + case Some(other) => + throw YAMLParseException.badDictValue( + Set("until", "expr", "eos"), other, path ++ List("repeat") + ) + } + } +} \ No newline at end of file diff --git a/shared/src/main/scala/io/kaitai/struct/format/ValidationSpec.scala b/shared/src/main/scala/io/kaitai/struct/format/ValidationSpec.scala new file mode 100644 index 000000000..4aaec38cb --- /dev/null +++ b/shared/src/main/scala/io/kaitai/struct/format/ValidationSpec.scala @@ -0,0 +1,32 @@ +package io.kaitai.struct.format + +import io.kaitai.struct.exprlang.{Ast, Expressions} + +sealed trait ValidationSpec + +case class ValidationEq(value: Ast.expr) extends ValidationSpec +case class ValidationRange(min: Option[Ast.expr], max: Option[Ast.expr]) extends ValidationSpec +case class ValidationAnyOf(values: List[Ast.expr]) extends ValidationSpec +case class ValidationExpr(checkExpr: Ast.expr) extends ValidationSpec + +object ValidationSpec { + def fromYaml(src: Any, path: List[String]): ValidationSpec = { + src match { + case value: String => + fromString(value, path) + case x: Boolean => + fromString(x.toString, path) + case x: Int => + fromString(x.toString, path) + case x: Long => + fromString(x.toString, path) +// case srcMap: Map[Any, Any] => +// fromMap(ParseUtils.anyMapToStrMap(srcMap, path), path) + case _ => + throw YAMLParseException.badType("string or map", src, path) + } + } + + def fromString(value: String, path: List[String]): ValidationSpec = + ValidationEq(Expressions.parse(value)) +} diff --git a/shared/src/main/scala/io/kaitai/struct/format/YAMLParseException.scala b/shared/src/main/scala/io/kaitai/struct/format/YAMLParseException.scala index 747f044fa..bf5829e00 100644 --- a/shared/src/main/scala/io/kaitai/struct/format/YAMLParseException.scala +++ b/shared/src/main/scala/io/kaitai/struct/format/YAMLParseException.scala @@ -1,6 +1,7 @@ package io.kaitai.struct.format import fastparse.StringReprOps +import io.kaitai.struct.Utils import io.kaitai.struct.datatype.DataType import io.kaitai.struct.exprlang.Expressions @@ -8,6 +9,9 @@ class YAMLParseException(val msg: String, val path: List[String]) extends RuntimeException(s"/${path.mkString("/")}: $msg", null) object YAMLParseException { + def noKey(path: List[String]): YAMLParseException = + new YAMLParseException(s"missing mandatory argument `${path.last}`", path) + def badType(expected: String, got: Any, path: List[String]): YAMLParseException = { val gotStr = got match { case null => "null" @@ -34,12 +38,38 @@ object YAMLParseException { def expression(epe: Expressions.ParseException, path: List[String]): YAMLParseException = { val f = epe.failure val pos = StringReprOps.prettyIndex(f.extra.input, f.index) + + // Try to diagnose most common errors and provide a friendly suggestion + val lookup2 = Utils.safeLookup(epe.src, f.index, 2) + val suggestion: String = (if (lookup2 == "&&") { + Some("and") + } else if (lookup2 == "||") { + Some("or") + } else { + None + }).map((x) => s", did you mean '$x'?").getOrElse("") + + f.extra.traced.expected + new YAMLParseException( - s"parsing expression '${epe.src}' failed on $pos, expected ${f.extra.traced.expected}", + s"parsing expression '${epe.src}' failed on $pos, " + + s"expected ${f.extra.traced.expected.replaceAll("\n", "\\n")}$suggestion", path ) } def exprType(expected: String, got: DataType, path: List[String]): YAMLParseException = new YAMLParseException(s"invalid type: expected $expected, got $got", path) + + def badProcess(got: String, path: List[String]): YAMLParseException = + new YAMLParseException(s"incorrect process expression `$got`", path) + + def invalidParamCount(paramSize: Int, argSize: Int, path: List[String]): YAMLParseException = + new YAMLParseException(s"parameter count mismatch: $paramSize declared, but $argSize used", path) + + def paramMismatch(idx: Int, argType: DataType, paramName: String, paramType: DataType, path: List[String]): YAMLParseException = + new YAMLParseException( + s"can't pass argument #$idx of type $argType into parameter `$paramName` of type $paramType", + path + ) } diff --git a/shared/src/main/scala/io/kaitai/struct/format/YAMLPath.scala b/shared/src/main/scala/io/kaitai/struct/format/YAMLPath.scala index 7c1dd465c..9282a6e66 100644 --- a/shared/src/main/scala/io/kaitai/struct/format/YAMLPath.scala +++ b/shared/src/main/scala/io/kaitai/struct/format/YAMLPath.scala @@ -1,5 +1,11 @@ package io.kaitai.struct.format +/** + * Common trait for all format parts that stores YAML path that corresponds + * to particular format part. Used to throw path-localized exceptions, i.e. + * [[YAMLParseException]] and [[io.kaitai.struct.precompile.ErrorInInput]], + * and implement better error messaging. + */ trait YAMLPath { def path: List[String] diff --git a/shared/src/main/scala/io/kaitai/struct/languages/CSharpCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/CSharpCompiler.scala index ef90af546..f9f9c1652 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/CSharpCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/CSharpCompiler.scala @@ -1,15 +1,15 @@ package io.kaitai.struct.languages import io.kaitai.struct._ -import io.kaitai.struct.datatype.DataType import io.kaitai.struct.datatype.DataType._ +import io.kaitai.struct.datatype._ import io.kaitai.struct.exprlang.Ast import io.kaitai.struct.exprlang.Ast.expr -import io.kaitai.struct.format.{RepeatUntil, _} +import io.kaitai.struct.format._ import io.kaitai.struct.languages.components._ -import io.kaitai.struct.translators.{CSharpTranslator, TypeDetector, TypeProvider} +import io.kaitai.struct.translators.{CSharpTranslator, TypeDetector} -class CSharpCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) +class CSharpCompiler(val typeProvider: ClassTypeProvider, config: RuntimeConfig) extends LanguageCompiler(typeProvider, config) with UpperCamelCaseClasses with ObjectOrientedLanguage @@ -18,28 +18,30 @@ class CSharpCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) with EveryReadIsExpression with UniversalDoc with FixedContentsUsingArrayByteLiteral + with SwitchIfOps with NoNeedForFullClassPath { import CSharpCompiler._ - val translator = new CSharpTranslator(typeProvider) + val translator = new CSharpTranslator(typeProvider, importList) override def indent: String = " " override def outFileName(topClassName: String): String = s"${type2class(topClassName)}.cs" + override def outImports(topClass: ClassSpec) = + importList.toList.map((x) => s"using $x;").mkString("", "\n", "\n") + override def fileHeader(topClassName: String): Unit = { - out.puts(s"// $headerComment") + outHeader.puts(s"// $headerComment") + outHeader.puts var ns = "Kaitai" if (!config.dotNetNamespace.isEmpty) ns = config.dotNetNamespace - out.puts - out.puts("using System;") - out.puts("using System.Collections.Generic;") - out.puts("using System.Linq;") - if (ns != "Kaitai") out.puts("using Kaitai;") - out.puts + if (ns != "Kaitai") + importList.add("Kaitai") + out.puts out.puts(s"namespace $ns") out.puts(s"{") out.inc @@ -56,60 +58,120 @@ class CSharpCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts(s"{") out.inc - out.puts(s"public static ${type2class(name)} FromFile(string fileName)") - out.puts(s"{") - out.inc - out.puts(s"return new ${type2class(name)}(new $kstreamName(fileName));") - out.dec - out.puts("}") + // `FromFile` is generated only for parameterless types + if (typeProvider.nowClass.params.isEmpty) { + out.puts(s"public static ${type2class(name)} FromFile(string fileName)") + out.puts(s"{") + out.inc + out.puts(s"return new ${type2class(name)}(new $kstreamName(fileName));") + out.dec + out.puts("}") + out.puts + } } override def classFooter(name: String): Unit = fileFooter(name) - override def classConstructorHeader(name: String, parentClassName: String, rootClassName: String): Unit = { - out.puts - out.puts(s"public ${type2class(name)}($kstreamName io, ${type2class(parentClassName)} parent = null, ${type2class(rootClassName)} root = null) : base(io)") + override def classConstructorHeader(name: String, parentType: DataType, rootClassName: String, isHybrid: Boolean, params: List[ParamDefSpec]): Unit = { + typeProvider.nowClass.meta.endian match { + case Some(_: CalcEndian) | Some(InheritedEndian) => + out.puts(s"private bool? ${privateMemberName(EndianIdentifier)};") + case _ => + // no _is_le variable + } + + val addEndian = if (isHybrid) ", bool? isLe = null" else "" + + val pIo = paramName(IoIdentifier) + val pParent = paramName(ParentIdentifier) + val pRoot = paramName(RootIdentifier) + + val paramsArg = Utils.join(params.map((p) => + s"${kaitaiType2NativeType(p.dataType)} ${paramName(p.id)}" + ), "", ", ", ", ") + + out.puts( + s"public ${type2class(name)}($paramsArg" + + s"$kstreamName $pIo, " + + s"${kaitaiType2NativeType(parentType)} $pParent = null, " + + s"${type2class(rootClassName)} $pRoot = null$addEndian) : base($pIo)" + ) out.puts(s"{") out.inc - out.puts(s"${privateMemberName(ParentIdentifier)} = parent;") + handleAssignmentSimple(ParentIdentifier, pParent) + + handleAssignmentSimple( + RootIdentifier, + if (name == rootClassName) s"$pRoot ?? this" else pRoot + ) + + if (isHybrid) + handleAssignmentSimple(EndianIdentifier, "isLe") + + // Store parameters passed to us + params.foreach((p) => handleAssignmentSimple(p.id, paramName(p.id))) + } + + override def classConstructorFooter: Unit = fileFooter(null) - if (name == rootClassName) - out.puts(s"${privateMemberName(RootIdentifier)} = root ?? this;") - else - out.puts(s"${privateMemberName(RootIdentifier)} = root;") + override def runRead(): Unit = + out.puts("_read();") - out.puts("_parse();") + override def runReadCalc(): Unit = { + out.puts + out.puts(s"if (${privateMemberName(EndianIdentifier)} == null) {") + out.inc + out.puts("throw new Exception(\"Unable to decide on endianness\");") + importList.add("System") + out.dec + out.puts(s"} else if (${privateMemberName(EndianIdentifier)} == true) {") + out.inc + out.puts("_readLE();") + out.dec + out.puts("} else {") + out.inc + out.puts("_readBE();") out.dec out.puts("}") - out.puts + } - out.puts("private void _parse()") + override def readHeader(endian: Option[FixedEndian], isEmpty: Boolean) = { + val readAccessAndType = if (!config.autoRead) { + "public" + } else { + "private" + } + val suffix = endian match { + case Some(e) => s"${e.toSuffix.toUpperCase}" + case None => "" + } + out.puts(s"$readAccessAndType void _read$suffix()") out.puts("{") out.inc } - override def classConstructorFooter: Unit = fileFooter(null) + override def readFooter(): Unit = fileFooter("") - override def attributeDeclaration(attrName: Identifier, attrType: DataType, condSpec: ConditionalSpec): Unit = { - out.puts(s"private ${kaitaiType2NativeType(attrType)} ${privateMemberName(attrName)};") + override def attributeDeclaration(attrName: Identifier, attrType: DataType, isNullable: Boolean): Unit = { + out.puts(s"private ${kaitaiType2NativeTypeNullable(attrType, isNullable)} ${privateMemberName(attrName)};") } - override def attributeReader(attrName: Identifier, attrType: DataType, condSpec: ConditionalSpec): Unit = { - out.puts(s"public ${kaitaiType2NativeType(attrType)} ${publicMemberName(attrName)} { get { return ${privateMemberName(attrName)}; } }") + override def attributeReader(attrName: Identifier, attrType: DataType, isNullable: Boolean): Unit = { + out.puts(s"public ${kaitaiType2NativeTypeNullable(attrType, isNullable)} ${publicMemberName(attrName)} { get { return ${privateMemberName(attrName)}; } }") } override def universalDoc(doc: DocSpec): Unit = { out.puts - doc.summary.foreach { (summary) => + doc.summary.foreach { summary => out.puts("/// ") out.putsLines("/// ", XMLUtils.escape(summary)) out.puts("/// ") } - if (doc.ref != NoRef) { + doc.ref.foreach { docRef => out.puts("/// ") - val refStr = doc.ref match { + val refStr = docRef match { case TextRef(text) => XMLUtils.escape(text) case ref: UrlRef => ref.toAhref } @@ -119,6 +181,18 @@ class CSharpCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } } + override def attrParseHybrid(leProc: () => Unit, beProc: () => Unit): Unit = { + out.puts(s"if (${privateMemberName(EndianIdentifier)} == true) {") + out.inc + leProc() + out.dec + out.puts("} else {") + out.inc + beProc() + out.dec + out.puts("}") + } + override def attrFixedContentsParse(attrName: Identifier, contents: String): Unit = out.puts(s"${privateMemberName(attrName)} = $normalIO.EnsureFixedContents($contents);") @@ -138,6 +212,11 @@ class CSharpCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) s"8 - (${expression(rotValue)})" } out.puts(s"$destName = $normalIO.ProcessRotateLeft($srcName, $expr, 1);") + case ProcessCustom(name, args) => + val procClass = types2class(name) + val procName = s"_process_${idToStr(varSrc)}" + out.puts(s"$procClass $procName = new $procClass(${args.map(expression).mkString(", ")});") + out.puts(s"$destName = $procName.Decode($srcName);") } } @@ -189,9 +268,14 @@ class CSharpCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def condIfFooter(expr: expr): Unit = fileFooter(null) override def condRepeatEosHeader(id: Identifier, io: String, dataType: DataType, needRaw: Boolean): Unit = { + importList.add("System.Collections.Generic") + if (needRaw) out.puts(s"${privateMemberName(RawIdentifier(id))} = new List();") out.puts(s"${privateMemberName(id)} = new ${kaitaiType2NativeType(ArrayType(dataType))}();") + out.puts("{") + out.inc + out.puts("var i = 0;") out.puts(s"while (!$io.IsEof) {") out.inc } @@ -200,13 +284,22 @@ class CSharpCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts(s"${privateMemberName(id)}.Add($expr);") } - override def condRepeatEosFooter: Unit = fileFooter(null) + override def condRepeatEosFooter: Unit = { + out.puts("i++;") + out.dec + out.puts("}") + out.dec + out.puts("}") + } override def condRepeatExprHeader(id: Identifier, io: String, dataType: DataType, needRaw: Boolean, repeatExpr: expr): Unit = { + importList.add("System.Collections.Generic") + if (needRaw) out.puts(s"${privateMemberName(RawIdentifier(id))} = new List((int) (${expression(repeatExpr)}));") out.puts(s"${privateMemberName(id)} = new ${kaitaiType2NativeType(ArrayType(dataType))}((int) (${expression(repeatExpr)}));") - out.puts(s"for (var i = 0; i < ${expression(repeatExpr)}; i++) {") + out.puts(s"for (var i = 0; i < ${expression(repeatExpr)}; i++)") + out.puts("{") out.inc } @@ -217,11 +310,14 @@ class CSharpCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def condRepeatExprFooter: Unit = fileFooter(null) override def condRepeatUntilHeader(id: Identifier, io: String, dataType: DataType, needRaw: Boolean, untilExpr: expr): Unit = { + importList.add("System.Collections.Generic") + if (needRaw) out.puts(s"${privateMemberName(RawIdentifier(id))} = new List();") out.puts(s"${privateMemberName(id)} = new ${kaitaiType2NativeType(ArrayType(dataType))}();") out.puts("{") out.inc + out.puts("var i = 0;") out.puts(s"${kaitaiType2NativeType(dataType)} ${translator.doName("_")};") out.puts("do {") out.inc @@ -239,20 +335,23 @@ class CSharpCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def condRepeatUntilFooter(id: Identifier, io: String, dataType: DataType, needRaw: Boolean, untilExpr: expr): Unit = { typeProvider._currentIteratorType = Some(dataType) + out.puts("i++;") out.dec out.puts(s"} while (!(${expression(untilExpr)}));") out.dec out.puts("}") } - override def handleAssignmentSimple(id: Identifier, expr: String): Unit = { + override def handleAssignmentSimple(id: Identifier, expr: String): Unit = out.puts(s"${privateMemberName(id)} = $expr;") - } - override def parseExpr(dataType: DataType, io: String): String = { + override def handleAssignmentTempVar(dataType: DataType, id: String, expr: String): Unit = + out.puts(s"${kaitaiType2NativeType(dataType)} $id = $expr;") + + override def parseExpr(dataType: DataType, assignType: DataType, io: String, defEndian: Option[FixedEndian]): String = { dataType match { case t: ReadableType => - s"$io.Read${Utils.capitalize(t.apiCall)}()" + s"$io.Read${Utils.capitalize(t.apiCall(defEndian))}()" case blt: BytesLimitType => s"$io.ReadBytes(${expression(blt.size)})" case _: BytesEosType => @@ -264,6 +363,7 @@ class CSharpCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) case BitsType(width: Int) => s"$io.ReadBitsInt($width)" case t: UserType => + val addParams = Utils.join(t.args.map((a) => translator.translate(a)), "", ", ", ", ") val addArgs = if (t.isOpaque) { "" } else { @@ -272,9 +372,13 @@ class CSharpCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) case Some(fp) => translator.translate(fp) case None => "this" } - s", $parent, ${privateMemberName(RootIdentifier)}" + val addEndian = t.classSpec.get.meta.endian match { + case Some(InheritedEndian) => s", ${privateMemberName(EndianIdentifier)}" + case _ => "" + } + s", $parent, ${privateMemberName(RootIdentifier)}$addEndian" } - s"new ${types2class(t.name)}($io$addArgs)" + s"new ${types2class(t.name)}($addParams$io$addArgs)" } } @@ -290,9 +394,23 @@ class CSharpCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) expr2 } + override def userTypeDebugRead(id: String): Unit = + out.puts(s"$id._read();") + + override def switchRequiresIfs(onType: DataType): Boolean = onType match { + case _: IntType | _: EnumType | _: StrType => false + case _ => true + } + + // + + val NAME_SWITCH_ON = Ast.expr.Name(Ast.identifier(Identifier.SWITCH_ON)) + override def switchStart(id: Identifier, on: Ast.expr): Unit = out.puts(s"switch (${expression(on)}) {") + override def switchCaseFirstStart(condition: Ast.expr): Unit = switchCaseStart(condition) + override def switchCaseStart(condition: Ast.expr): Unit = { out.puts(s"case ${expression(condition)}: {") out.inc @@ -312,13 +430,62 @@ class CSharpCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def switchEnd(): Unit = out.puts("}") - override def instanceDeclaration(attrName: InstanceIdentifier, attrType: DataType, condSpec: ConditionalSpec): Unit = { + // + + // + + override def switchIfStart(id: Identifier, on: Ast.expr, onType: DataType): Unit = { + out.puts("{") + out.inc + out.puts(s"${kaitaiType2NativeType(onType)} ${expression(NAME_SWITCH_ON)} = ${expression(on)};") + } + + def switchCmpExpr(condition: Ast.expr): String = + expression( + Ast.expr.Compare( + NAME_SWITCH_ON, + Ast.cmpop.Eq, + condition + ) + ) + + override def switchIfCaseFirstStart(condition: Ast.expr): Unit = { + out.puts(s"if (${switchCmpExpr(condition)})") + out.puts("{") + out.inc + } + + override def switchIfCaseStart(condition: Ast.expr): Unit = { + out.puts(s"else if (${switchCmpExpr(condition)})") + out.puts("{") + out.inc + } + + override def switchIfCaseEnd(): Unit = { + out.dec + out.puts("}") + } + + override def switchIfElseStart(): Unit = { + out.puts("else") + out.puts("{") + out.inc + } + + override def switchIfEnd(): Unit = { + out.dec + out.puts("}") + } + + // + + override def instanceDeclaration(attrName: InstanceIdentifier, attrType: DataType, isNullable: Boolean): Unit = { out.puts(s"private bool ${flagForInstName(attrName)};") - out.puts(s"private ${kaitaiType2NativeType(attrType)} ${privateMemberName(attrName)};") + out.puts(s"private ${kaitaiType2NativeTypeNullable(attrType, isNullable)} ${privateMemberName(attrName)};") } - override def instanceHeader(className: String, instName: InstanceIdentifier, dataType: DataType): Unit = { - out.puts(s"public ${kaitaiType2NativeType(dataType)} ${publicMemberName(instName)}") + override def instanceHeader(className: String, instName: InstanceIdentifier, dataType: DataType, isNullable: Boolean): Unit = { + out.puts(s"public ${kaitaiType2NativeTypeNullable(dataType, isNullable)} ${publicMemberName(instName)}") out.puts("{") out.inc out.puts("get") @@ -333,18 +500,18 @@ class CSharpCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts("}") } - override def instanceCheckCacheAndReturn(instName: InstanceIdentifier): Unit = { + override def instanceCheckCacheAndReturn(instName: InstanceIdentifier, dataType: DataType): Unit = { out.puts(s"if (${flagForInstName(instName)})") out.inc - instanceReturn(instName) + instanceReturn(instName, dataType) out.dec } - override def instanceReturn(instName: InstanceIdentifier): Unit = { + override def instanceReturn(instName: InstanceIdentifier, attrType: DataType): Unit = { out.puts(s"return ${privateMemberName(instName)};") } - override def instanceCalculate(instName: InstanceIdentifier, dataType: DataType, value: expr): Unit = + override def instanceCalculate(instName: Identifier, dataType: DataType, value: expr): Unit = // Perform explicit cast as unsigned integers can't be directly assigned to the default int type handleAssignmentSimple(instName, s"(${kaitaiType2NativeType(dataType)}) (${expression(value)})") @@ -392,11 +559,18 @@ class CSharpCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) case _ => s"_${idToStr(id)}" } } + + override def localTemporaryName(id: Identifier): String = s"_t_${idToStr(id)}" + + override def paramName(id: Identifier): String = s"p_${idToStr(id)}" + + override def ksErrorName(err: KSError): String = CSharpCompiler.ksErrorName(err) } object CSharpCompiler extends LanguageCompilerStatic with StreamStructNames - with UpperCamelCaseClasses { + with UpperCamelCaseClasses + with ExceptionNames { override def getCompiler( tp: ClassTypeProvider, config: RuntimeConfig @@ -433,7 +607,7 @@ object CSharpCompiler extends LanguageCompilerStatic case _: BytesType => "byte[]" case AnyType => "object" - case KaitaiStructType => kstructName + case KaitaiStructType | CalcKaitaiStructType => kstructName case KaitaiStreamType => kstreamName case t: UserType => types2class(t.name) @@ -441,14 +615,30 @@ object CSharpCompiler extends LanguageCompilerStatic case ArrayType(inType) => s"List<${kaitaiType2NativeType(inType)}>" - case SwitchType(_, cases) => kaitaiType2NativeType(TypeDetector.combineTypes(cases.values)) + case st: SwitchType => kaitaiType2NativeType(st.combinedType) + } + } + + def kaitaiType2NativeTypeNullable(t: DataType, isNullable: Boolean): String = { + val r = kaitaiType2NativeType(t) + if (isNullable) { + t match { + case _: NumericType | _: BooleanType => s"$r?" + case _ => r + } + } else { + r } } - def types2class(names: List[String]) = names.map(x => type2class(x)).mkString(".") + def types2class(typeName: Ast.typeId): String = + // FIXME: handle absolute + types2class(typeName.names) + def types2class(names: Iterable[String]) = names.map(type2class).mkString(".") override def kstructName = "KaitaiStruct" override def kstreamName = "KaitaiStream" + override def ksErrorName(err: KSError): String = err.name override def type2class(name: String): String = Utils.upperCamelCase(name) } diff --git a/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala index 0587bccd0..c8f3ee23f 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala @@ -1,37 +1,47 @@ package io.kaitai.struct.languages +import io.kaitai.struct.CppRuntimeConfig._ import io.kaitai.struct._ -import io.kaitai.struct.datatype.DataType import io.kaitai.struct.datatype.DataType._ +import io.kaitai.struct.datatype._ import io.kaitai.struct.exprlang.Ast import io.kaitai.struct.exprlang.Ast.expr import io.kaitai.struct.format._ import io.kaitai.struct.languages.components._ -import io.kaitai.struct.translators.{CppTranslator, TypeDetector, TypeProvider} +import io.kaitai.struct.translators.{CppTranslator, TypeDetector} class CppCompiler( - typeProvider: ClassTypeProvider, + val typeProvider: ClassTypeProvider, config: RuntimeConfig ) extends LanguageCompiler(typeProvider, config) with ObjectOrientedLanguage with AllocateAndStoreIO with FixedContentsUsingArrayByteLiteral with UniversalDoc + with SwitchIfOps with EveryReadIsExpression { import CppCompiler._ - override val translator = new CppTranslator(typeProvider) + val importListSrc = new ImportList + val importListHdr = new ImportList + + override val translator = new CppTranslator(typeProvider, importListSrc, config) + val outSrcHeader = new StringLanguageOutputWriter(indent) + val outHdrHeader = new StringLanguageOutputWriter(indent) val outSrc = new StringLanguageOutputWriter(indent) val outHdr = new StringLanguageOutputWriter(indent) override def results(topClass: ClassSpec): Map[String, String] = { val fn = topClass.nameAsStr Map( - s"$fn.cpp" -> outSrc.result, - s"$fn.h" -> outHdr.result + s"$fn.cpp" -> (outSrcHeader.result + importListToStr(importListSrc) + outSrc.result), + s"$fn.h" -> (outHdrHeader.result + importListToStr(importListHdr) + outHdr.result) ) } + private def importListToStr(importList: ImportList): String = + importList.toList.map((x) => s"#include <$x>").mkString("", "\n", "\n") + sealed trait AccessMode case object PrivateAccess extends AccessMode case object PublicAccess extends AccessMode @@ -42,25 +52,32 @@ class CppCompiler( override def outFileName(topClassName: String): String = topClassName override def fileHeader(topClassName: String): Unit = { - outSrc.puts(s"// $headerComment") - outSrc.puts - outSrc.puts("#include \"" + outFileName(topClassName) + ".h\"") - outSrc.puts - outSrc.puts("#include ") - outSrc.puts("#include ") - - outHdr.puts(s"#ifndef ${defineName(topClassName)}") - outHdr.puts(s"#define ${defineName(topClassName)}") - outHdr.puts - outHdr.puts(s"// $headerComment") - outHdr.puts - outHdr.puts("#include ") - outHdr.puts("#include ") - outHdr.puts - outHdr.puts("#include ") - outHdr.puts("#include ") // TODO: add only if required - outHdr.puts("#include ") // TODO: add only if required - outHdr.puts("#include ") // TODO: add only if required + outSrcHeader.puts(s"// $headerComment") + outSrcHeader.puts + outSrcHeader.puts("#include ") + outSrcHeader.puts("#include \"" + outFileName(topClassName) + ".h\"") + outSrcHeader.puts + + if (config.cppConfig.usePragmaOnce) { + outHdrHeader.puts("#pragma once") + } else { + outHdrHeader.puts(s"#ifndef ${defineName(topClassName)}") + outHdrHeader.puts(s"#define ${defineName(topClassName)}") + } + outHdrHeader.puts + outHdrHeader.puts(s"// $headerComment") + outHdrHeader.puts + outHdrHeader.puts("#include \"kaitai/kaitaistruct.h\"") + outHdrHeader.puts + + importListHdr.add("stdint.h") + + config.cppConfig.pointers match { + case SharedPointers | UniqueAndRawPointers => + importListHdr.add("memory") + case RawPointers => + // no extra includes + } // API compatibility check val minVer = KSVersion.minimalRuntime.toInt @@ -71,11 +88,27 @@ class CppCompiler( KSVersion.minimalRuntime + " or later is required\"" ) outHdr.puts("#endif") + + config.cppConfig.namespace.foreach { (namespace) => + outSrc.puts(s"namespace $namespace {") + outSrc.inc + outHdr.puts(s"namespace $namespace {") + outHdr.inc + } } override def fileFooter(topClassName: String): Unit = { - outHdr.puts - outHdr.puts(s"#endif // ${defineName(topClassName)}") + config.cppConfig.namespace.foreach { (_) => + outSrc.dec + outSrc.puts("}") + outHdr.dec + outHdr.puts("}") + } + + if (!config.cppConfig.usePragmaOnce) { + outHdr.puts + outHdr.puts(s"#endif // ${defineName(topClassName)}") + } } override def opaqueClassDeclaration(classSpec: ClassSpec): Unit = { @@ -84,8 +117,15 @@ class CppCompiler( } override def classHeader(name: List[String]): Unit = { + val className = types2class(List(name.last)) + + val extraInherits = config.cppConfig.pointers match { + case RawPointers | UniqueAndRawPointers => "" + case SharedPointers => s", std::enable_shared_from_this<$className>" + } + outHdr.puts - outHdr.puts(s"class ${types2class(List(name.last))} : public $kstructName {") + outHdr.puts(s"class $className : public $kstructName$extraInherits {") outHdr.inc accessMode = PrivateAccess ensureMode(PublicAccess) @@ -113,27 +153,73 @@ class CppCompiler( outHdr.puts(s"class ${types2class(name)};") } - override def classConstructorHeader(name: List[String], parentClassName: List[String], rootClassName: List[String]): Unit = { + override def classConstructorHeader(name: List[String], parentType: DataType, rootClassName: List[String], isHybrid: Boolean, params: List[ParamDefSpec]): Unit = { + val (endianSuffixHdr, endianSuffixSrc) = if (isHybrid) { + (", int p_is_le = -1", ", int p_is_le") + } else { + ("", "") + } + + val paramsArg = Utils.join(params.map((p) => + s"${kaitaiType2NativeType(p.dataType)} ${paramName(p.id)}" + ), "", ", ", ", ") + + val classNameBrief = types2class(List(name.last)) + + // Parameter names + val pIo = paramName(IoIdentifier) + val pParent = paramName(ParentIdentifier) + val pRoot = paramName(RootIdentifier) + + // Types + val tIo = kaitaiType2NativeType(KaitaiStreamType) + val tParent = kaitaiType2NativeType(parentType) + val tRoot = kaitaiType2NativeType(CalcUserType(rootClassName, None)) + outHdr.puts - outHdr.puts(s"${types2class(List(name.last))}(" + - s"$kstreamName* p_io, " + - s"${types2class(parentClassName)}* p_parent = 0, " + - s"${types2class(rootClassName)}* p_root = 0);" + outHdr.puts(s"$classNameBrief($paramsArg" + + s"$tIo $pIo, " + + s"$tParent $pParent = $nullPtr, " + + s"$tRoot $pRoot = $nullPtr$endianSuffixHdr);" ) outSrc.puts - outSrc.puts(s"${types2class(name)}::${types2class(List(name.last))}(" + - s"$kstreamName *p_io, " + - s"${types2class(parentClassName)} *p_parent, " + - s"${types2class(rootClassName)} *p_root) : $kstructName(p_io) {" + outSrc.puts(s"${types2class(name)}::$classNameBrief($paramsArg" + + s"$tIo $pIo, " + + s"$tParent $pParent, " + + s"$tRoot $pRoot$endianSuffixSrc) : $kstructName($pIo) {" ) outSrc.inc - handleAssignmentSimple(ParentIdentifier, "p_parent") + + // In shared pointers mode, this is required to be able to work with shared pointers to this + // in a constructor. This is obviously a hack and not a good practice. + // https://forum.libcinder.org/topic/solution-calling-shared-from-this-in-the-constructor + if (config.cppConfig.pointers == CppRuntimeConfig.SharedPointers) { + outSrc.puts(s"const auto weakPtrTrick = std::shared_ptr<$classNameBrief>(this, []($classNameBrief*){});") + } + + handleAssignmentSimple(ParentIdentifier, pParent) handleAssignmentSimple(RootIdentifier, if (name == rootClassName) { - "this" + config.cppConfig.pointers match { + case RawPointers | UniqueAndRawPointers => "this" + case SharedPointers => "shared_from_this()" + } } else { - "p_root" + pRoot }) + + typeProvider.nowClass.meta.endian match { + case Some(_: CalcEndian) | Some(InheritedEndian) => + ensureMode(PrivateAccess) + outHdr.puts("int m__is_le;") + handleAssignmentSimple(EndianIdentifier, if (isHybrid) "p_is_le" else "-1") + ensureMode(PublicAccess) + case _ => + // no _is_le variable + } + + // Store parameters passed to us + params.foreach((p) => handleAssignmentSimple(p.id, paramName(p.id))) } override def classConstructorFooter: Unit = { @@ -141,7 +227,7 @@ class CppCompiler( outSrc.puts("}") } - override def classDestructorHeader(name: List[String], parentTypeName: List[String], topClassName: List[String]): Unit = { + override def classDestructorHeader(name: List[String], parentType: DataType, topClassName: List[String]): Unit = { outHdr.puts(s"~${types2class(List(name.last))}();") outSrc.puts @@ -151,10 +237,54 @@ class CppCompiler( override def classDestructorFooter = classConstructorFooter - override def attributeDeclaration(attrName: Identifier, attrType: DataType, condSpec: ConditionalSpec): Unit = { + override def runRead(): Unit = { + outSrc.puts("_read();") + } + + override def runReadCalc(): Unit = { + outSrc.puts + outSrc.puts("if (m__is_le == -1) {") + outSrc.inc + importListSrc.add("kaitai/exceptions.h") + outSrc.puts(s"throw ${ksErrorName(UndecidedEndiannessError)}" + + "(\"" + typeProvider.nowClass.path.mkString("/", "/", "") + "\");") + outSrc.dec + outSrc.puts("} else if (m__is_le == 1) {") + outSrc.inc + outSrc.puts("_read_le();") + outSrc.dec + outSrc.puts("} else {") + outSrc.inc + outSrc.puts("_read_be();") + outSrc.dec + outSrc.puts("}") + } + + override def readHeader(endian: Option[FixedEndian], isEmpty: Boolean): Unit = { + val suffix = endian match { + case Some(e) => s"_${e.toSuffix}" + case None => "" + } + + ensureMode(if (config.autoRead) PrivateAccess else PublicAccess) + + outHdr.puts(s"void _read$suffix();") + outSrc.puts + outSrc.puts(s"void ${types2class(typeProvider.nowClass.name)}::_read$suffix() {") + outSrc.inc + } + + override def readFooter(): Unit = { + outSrc.dec + outSrc.puts("}") + + ensureMode(PublicAccess) + } + + override def attributeDeclaration(attrName: Identifier, attrType: DataType, isNullable: Boolean): Unit = { ensureMode(PrivateAccess) outHdr.puts(s"${kaitaiType2NativeType(attrType)} ${privateMemberName(attrName)};") - declareNullFlag(attrName, condSpec) + declareNullFlag(attrName, isNullable) } def ensureMode(newMode: AccessMode): Unit = { @@ -170,9 +300,9 @@ class CppCompiler( } } - override def attributeReader(attrName: Identifier, attrType: DataType, condSpec: ConditionalSpec): Unit = { + override def attributeReader(attrName: Identifier, attrType: DataType, isNullable: Boolean): Unit = { ensureMode(PublicAccess) - outHdr.puts(s"${kaitaiType2NativeType(attrType)} ${publicMemberName(attrName)}() const { return ${privateMemberName(attrName)}; }") + outHdr.puts(s"${kaitaiType2NativeType(attrType.asNonOwning)} ${publicMemberName(attrName)}() const { return ${nonOwningPointer(attrName, attrType)}; }") } override def universalDoc(doc: DocSpec): Unit = { @@ -182,94 +312,137 @@ class CppCompiler( outHdr.puts outHdr.puts( "/**") - doc.summary.foreach((docStr) => outHdr.putsLines(" * ", docStr)) + doc.summary.foreach(docStr => outHdr.putsLines(" * ", docStr)) - doc.ref match { + doc.ref.foreach { case TextRef(text) => outHdr.putsLines(" * ", s"\\sa $text") case UrlRef(url, text) => - outHdr.putsLines(" * ", s"\\sa $text") - case NoRef => - // nothing to output + outHdr.putsLines(" * ", s"\\sa $url $text") } outHdr.puts( " */") } - override def attrDestructor(attr: AttrLikeSpec, id: Identifier): Unit = { - val t = attr.dataTypeComposite - - val checkFlags = attr match { - case is: InstanceSpec => - val dataType = is.dataTypeComposite - dataType match { - case ut: UserType => - val checkLazy = calculatedFlagForName(id.asInstanceOf[InstanceIdentifier]) - val checkNull = if (is.cond.ifExpr.isDefined) { - s" && !${nullFlagForName(id)}" - } else { - "" - } - outSrc.puts(s"if ($checkLazy$checkNull) {") - outSrc.inc - true - case _ => - false - } - case as: AttrSpec => - as.dataType match { - case ut: UserType => - if (as.cond.ifExpr.isDefined) { - outSrc.puts(s"if (!${nullFlagForName(id)}) {") - outSrc.inc - true - } else { - false - } - case _ => - false - } + override def attrInit(attr: AttrLikeSpec): Unit = { + attr.dataTypeComposite match { + case _: UserType | _: ArrayType | KaitaiStreamType => + // data type will be pointer to user type, std::vector or stream, so we need to init it + outSrc.puts(s"${privateMemberName(attr.id)} = $nullPtr;") + case _ => + // no init required for value types } + } - t match { - case ArrayType(_: UserTypeFromBytes) => - outSrc.puts(s"delete ${privateMemberName(RawIdentifier(id))};") - case _ => - // no cleanup needed + override def attrDestructor(attr: AttrLikeSpec, id: Identifier): Unit = { + val checkLazy = if (attr.isLazy) { + Some(calculatedFlagForName(id)) + } else { + None } - t match { - case ArrayType(el: UserType) => - val arrVar = privateMemberName(id) - outSrc.puts(s"for (std::vector<${kaitaiType2NativeType(el)}>::iterator it = $arrVar->begin(); it != $arrVar->end(); ++it) {") - outSrc.inc - outSrc.puts("delete *it;") - outSrc.dec - outSrc.puts("}") - case _ => - // no cleanup needed + val checkNull = if (attr.isNullableSwitchRaw) { + Some(s"!${nullFlagForName(id)}") + } else { + None } - t match { - case _: UserTypeFromBytes => - outSrc.puts(s"delete ${privateMemberName(IoStorageIdentifier(RawIdentifier(id)))};") - case _ => - // no cleanup needed + val checks: List[String] = List(checkLazy, checkNull).flatten + + if (checks.nonEmpty) { + outSrc.puts(s"if (${checks.mkString(" && ")}) {") + outSrc.inc } - t match { - case _: UserType | _: ArrayType => - outSrc.puts(s"delete ${privateMemberName(id)};") - case _ => - // no cleanup needed + val (innerType, hasRaw) = attr.dataType match { + case ut: UserTypeFromBytes => (ut, true) + case st: SwitchType => (st.combinedType, st.hasSize) + case t => (t, false) } - if (checkFlags) { + destructMember(id, innerType, attr.isArray, hasRaw, hasRaw) + + if (checks.nonEmpty) { outSrc.dec outSrc.puts("}") } } + def destructMember(id: Identifier, innerType: DataType, isArray: Boolean, hasRaw: Boolean, hasIO: Boolean): Unit = { + if (isArray) { + if (config.cppConfig.pointers == CppRuntimeConfig.RawPointers) { + // raw is std::vector*, no need to delete its contents, but we + // need to clean up the vector pointer itself + if (hasRaw) + outSrc.puts(s"delete ${privateMemberName(RawIdentifier(id))};") + + // IO is std::vector*, needs destruction of both members + // and the vector pointer itself + if (hasIO) { + val ioVar = privateMemberName(IoStorageIdentifier(RawIdentifier(id))) + destructVector(s"$kstreamName*", ioVar) + outSrc.puts(s"delete $ioVar;") + } + + // main member contents + if (needsDestruction(innerType)) { + val arrVar = privateMemberName(id) + + // C++ specific substitution: AnyType results from generic struct + raw bytes + // so we would assume that only generic struct needs to be cleaned up + val realType = innerType match { + case AnyType => KaitaiStructType + case _ => innerType + } + + destructVector(kaitaiType2NativeType(realType), arrVar) + } + + // main member is a std::vector of something, always needs destruction + outSrc.puts(s"delete ${privateMemberName(id)};") + } + } else { + // raw is just a string, no need to cleanup => we ignore `hasRaw` + + // but hasIO is important + if (hasIO) + outSrc.puts(s"delete ${privateMemberName(IoStorageIdentifier(RawIdentifier(id)))};") + + if (config.cppConfig.pointers == CppRuntimeConfig.RawPointers && needsDestruction(innerType)) + outSrc.puts(s"delete ${privateMemberName(id)};") + } + } + + def needsDestruction(t: DataType): Boolean = t match { + case _: UserType | _: ArrayType | KaitaiStructType | AnyType => true + case _ => false + } + + /** + * Generates std::vector contents destruction loop. + * @param elType element type, i.e. XXX in `std::vector<XXX>` + * @param arrVar variable name that holds pointer to std::vector + */ + def destructVector(elType: String, arrVar: String): Unit = { + outSrc.puts(s"for (std::vector<$elType>::iterator it = $arrVar->begin(); it != $arrVar->end(); ++it) {") + outSrc.inc + outSrc.puts("delete *it;") + outSrc.dec + outSrc.puts("}") + } + + override def attrParseHybrid(leProc: () => Unit, beProc: () => Unit): Unit = { + outSrc.puts("if (m__is_le == 1) {") + outSrc.inc + leProc() + outSrc.dec + outSrc.puts("} else {") + outSrc.inc + beProc() + outSrc.dec + outSrc.puts("}") + } + override def attrFixedContentsParse(attrName: Identifier, contents: String): Unit = outSrc.puts(s"${privateMemberName(attrName)} = $normalIO->ensure_fixed_contents($contents);") @@ -293,13 +466,20 @@ class CppCompiler( s"8 - (${expression(rotValue)})" } outSrc.puts(s"$destName = $kstreamName::process_rotate_left($srcName, $expr);") + case ProcessCustom(name, args) => + val procClass = name.map((x) => type2class(x)).mkString("::") + val procName = s"_process_${idToStr(varSrc)}" + + importListSrc.add(name.last + ".h") + + outSrc.puts(s"$procClass $procName(${args.map(expression).mkString(", ")});") + outSrc.puts(s"$destName = $procName.decode($srcName);") } } - override def allocateIO(varName: Identifier, rep: RepeatSpec): IoStorageIdentifier = { - val memberName = privateMemberName(varName) - - val ioName = IoStorageIdentifier(varName) + override def allocateIO(id: Identifier, rep: RepeatSpec): String = { + val memberName = privateMemberName(id) + val ioId = IoStorageIdentifier(id) val args = rep match { case RepeatEos | RepeatExpr(_) => s"$memberName->at($memberName->size() - 1)" @@ -307,7 +487,19 @@ class CppCompiler( case NoRepeat => memberName } - outSrc.puts(s"${privateMemberName(ioName)} = new $kstreamName($args);") + val newStream = s"new $kstreamName($args)" + + val ioName = rep match { + case NoRepeat => + outSrc.puts(s"${privateMemberName(ioId)} = $newStream;") + privateMemberName(ioId) + case _ => + val localIO = s"io_${idToStr(id)}" + outSrc.puts(s"$kstreamName* $localIO = $newStream;") + outSrc.puts(s"${privateMemberName(ioId)}->push_back($localIO);") + localIO + } + ioName } @@ -351,37 +543,53 @@ class CppCompiler( } override def condRepeatEosHeader(id: Identifier, io: String, dataType: DataType, needRaw: Boolean): Unit = { - if (needRaw) - outSrc.puts(s"${privateMemberName(RawIdentifier(id))} = new std::vector();") - outSrc.puts(s"${privateMemberName(id)} = new std::vector<${kaitaiType2NativeType(dataType)}>();") + importListHdr.add("vector") + + if (needRaw) { + outSrc.puts(s"${privateMemberName(RawIdentifier(id))} = ${newVector(CalcBytesType)};") + outSrc.puts(s"${privateMemberName(IoStorageIdentifier(RawIdentifier(id)))} = ${newVector(KaitaiStreamType)};") + } + outSrc.puts(s"${privateMemberName(id)} = ${newVector(dataType)};") + outSrc.puts("{") + outSrc.inc + outSrc.puts("int i = 0;") outSrc.puts(s"while (!$io->is_eof()) {") outSrc.inc } override def handleAssignmentRepeatEos(id: Identifier, expr: String): Unit = { - outSrc.puts(s"${privateMemberName(id)}->push_back($expr);") + outSrc.puts(s"${privateMemberName(id)}->push_back(${stdMoveWrap(expr)});") } override def condRepeatEosFooter: Unit = { + outSrc.puts("i++;") + outSrc.dec + outSrc.puts("}") outSrc.dec outSrc.puts("}") } override def condRepeatExprHeader(id: Identifier, io: String, dataType: DataType, needRaw: Boolean, repeatExpr: Ast.expr): Unit = { + importListHdr.add("vector") + val lenVar = s"l_${idToStr(id)}" outSrc.puts(s"int $lenVar = ${expression(repeatExpr)};") if (needRaw) { - outSrc.puts(s"${privateMemberName(RawIdentifier(id))} = new std::vector();") - outSrc.puts(s"${privateMemberName(RawIdentifier(id))}->reserve($lenVar);") + val rawId = privateMemberName(RawIdentifier(id)) + outSrc.puts(s"$rawId = ${newVector(CalcBytesType)};") + outSrc.puts(s"$rawId->reserve($lenVar);") + val ioId = privateMemberName(IoStorageIdentifier(RawIdentifier(id))) + outSrc.puts(s"$ioId = ${newVector(KaitaiStreamType)};") + outSrc.puts(s"$ioId->reserve($lenVar);") } - outSrc.puts(s"${privateMemberName(id)} = new std::vector<${kaitaiType2NativeType(dataType)}>();") + outSrc.puts(s"${privateMemberName(id)} = ${newVector(dataType)};") outSrc.puts(s"${privateMemberName(id)}->reserve($lenVar);") outSrc.puts(s"for (int i = 0; i < $lenVar; i++) {") outSrc.inc } override def handleAssignmentRepeatExpr(id: Identifier, expr: String): Unit = { - outSrc.puts(s"${privateMemberName(id)}->push_back($expr);") + outSrc.puts(s"${privateMemberName(id)}->push_back(${stdMoveWrap(expr)});") } override def condRepeatExprFooter: Unit = { @@ -390,28 +598,49 @@ class CppCompiler( } override def condRepeatUntilHeader(id: Identifier, io: String, dataType: DataType, needRaw: Boolean, untilExpr: expr): Unit = { - if (needRaw) - outSrc.puts(s"${privateMemberName(RawIdentifier(id))} = new std::vector();") - outSrc.puts(s"${privateMemberName(id)} = new std::vector<${kaitaiType2NativeType(dataType)}>();") + importListHdr.add("vector") + + if (needRaw) { + outSrc.puts(s"${privateMemberName(RawIdentifier(id))} = ${newVector(CalcBytesType)};") + outSrc.puts(s"${privateMemberName(IoStorageIdentifier(RawIdentifier(id)))} = ${newVector(KaitaiStreamType)};") + } + outSrc.puts(s"${privateMemberName(id)} = ${newVector(dataType)};") outSrc.puts("{") outSrc.inc - outSrc.puts(s"${kaitaiType2NativeType(dataType)} ${translator.doName("_")};") + outSrc.puts("int i = 0;") + outSrc.puts(s"${kaitaiType2NativeType(dataType.asNonOwning)} ${translator.doName("_")};") outSrc.puts("do {") outSrc.inc } + private val ReStdUniquePtr = "^std::unique_ptr<(.*?)>\\((.*?)\\)$".r + override def handleAssignmentRepeatUntil(id: Identifier, expr: String, isRaw: Boolean): Unit = { val (typeDecl, tempVar) = if (isRaw) { ("std::string ", translator.doName(Identifier.ITERATOR2)) } else { ("", translator.doName(Identifier.ITERATOR)) } - outSrc.puts(s"$typeDecl$tempVar = $expr;") - outSrc.puts(s"${privateMemberName(id)}->push_back($tempVar);") + + val (wrappedTempVar, rawPtrExpr) = if (config.cppConfig.pointers == UniqueAndRawPointers) { + expr match { + case ReStdUniquePtr(cppClass, innerExpr) => + (s"std::move(std::unique_ptr<$cppClass>($tempVar))", innerExpr) + case _ => + (tempVar, expr) + } + } else { + (tempVar, expr) + } + + outSrc.puts(s"$typeDecl$tempVar = $rawPtrExpr;") + + outSrc.puts(s"${privateMemberName(id)}->push_back($wrappedTempVar);") } override def condRepeatUntilFooter(id: Identifier, io: String, dataType: DataType, needRaw: Boolean, untilExpr: expr): Unit = { typeProvider._currentIteratorType = Some(dataType) + outSrc.puts("i++;") outSrc.dec outSrc.puts(s"} while (!(${expression(untilExpr)}));") outSrc.dec @@ -422,10 +651,13 @@ class CppCompiler( outSrc.puts(s"${privateMemberName(id)} = $expr;") } - override def parseExpr(dataType: DataType, io: String): String = { + override def handleAssignmentTempVar(dataType: DataType, id: String, expr: String): Unit = + outSrc.puts(s"${kaitaiType2NativeType(dataType)} $id = $expr;") + + override def parseExpr(dataType: DataType, assignType: DataType, io: String, defEndian: Option[FixedEndian]): String = { dataType match { case t: ReadableType => - s"$io->read_${t.apiCall}()" + s"$io->read_${t.apiCall(defEndian)}()" case blt: BytesLimitType => s"$io->read_bytes(${expression(blt.size)})" case _: BytesEosType => @@ -437,17 +669,47 @@ class CppCompiler( case BitsType(width: Int) => s"$io->read_bits_int($width)" case t: UserType => + val addParams = Utils.join(t.args.map((a) => translator.translate(a)), "", ", ", ", ") val addArgs = if (t.isOpaque) { "" } else { val parent = t.forcedParent match { - case Some(USER_TYPE_NO_PARENT) => "0" + case Some(USER_TYPE_NO_PARENT) => nullPtr case Some(fp) => translator.translate(fp) - case None => "this" + case None => + config.cppConfig.pointers match { + case RawPointers | UniqueAndRawPointers => "this" + case SharedPointers => s"shared_from_this()" + } + } + val addEndian = t.classSpec.get.meta.endian match { + case Some(InheritedEndian) => ", m__is_le" + case _ => "" } - s", $parent, ${privateMemberName(RootIdentifier)}" + s", $parent, ${privateMemberName(RootIdentifier)}$addEndian" + } + config.cppConfig.pointers match { + case RawPointers => + s"new ${types2class(t.name)}($addParams$io$addArgs)" + case SharedPointers => + s"std::make_shared<${types2class(t.name)}>($addParams$io$addArgs)" + case UniqueAndRawPointers => + importListSrc.add("memory") + // C++14 + //s"std::make_unique<${types2class(t.name)}>($addParams$io$addArgs)" + s"std::unique_ptr<${types2class(t.name)}>(new ${types2class(t.name)}($addParams$io$addArgs))" } - s"new ${types2class(t.name)}($io$addArgs)" + } + } + + def newVector(elType: DataType): String = { + val cppElType = kaitaiType2NativeType(elType) + config.cppConfig.pointers match { + case RawPointers => + s"new std::vector<$cppElType>()" + case UniqueAndRawPointers => + s"std::unique_ptr>(new std::vector<$cppElType>())" + // TODO: C++14 with std::make_unique } } @@ -463,93 +725,95 @@ class CppCompiler( expr2 } - /** - * Designates switch mode. If false, we're doing real switch-case for this - * attribute. If true, we're doing if-based emulation. - */ - var switchIfs = false + override def userTypeDebugRead(id: String): Unit = + outSrc.puts(s"$id->_read();") - override def switchStart(id: Identifier, on: Ast.expr): Unit = { - val onType = translator.detectType(on) + override def switchRequiresIfs(onType: DataType): Boolean = onType match { + case _: IntType | _: EnumType => false + case _ => true + } - // Determine switching mode for this construct based on type - switchIfs = onType match { - case _: IntType | _: EnumType => false - case _ => true - } + // - if (switchIfs) { - outSrc.puts("{") - outSrc.inc - outSrc.puts(s"${kaitaiType2NativeType(onType)} on = ${expression(on)};") - } else { - outSrc.puts(s"switch (${expression(on)}) {") - } - } + override def switchStart(id: Identifier, on: Ast.expr): Unit = + outSrc.puts(s"switch (${expression(on)}) {") override def switchCaseFirstStart(condition: Ast.expr): Unit = { - if (switchIfs) { - outSrc.puts(s"if (on == ${expression(condition)}) {") - outSrc.inc - } else { - outSrc.puts(s"case ${expression(condition)}:") - outSrc.inc - } + outSrc.puts(s"case ${expression(condition)}: {") + outSrc.inc } override def switchCaseStart(condition: Ast.expr): Unit = { - if (switchIfs) { - outSrc.puts(s"else if (on == ${expression(condition)}) {") - outSrc.inc - } else { - outSrc.puts(s"case ${expression(condition)}:") - outSrc.inc - } + outSrc.puts(s"case ${expression(condition)}: {") + outSrc.inc } override def switchCaseEnd(): Unit = { - if (switchIfs) { - outSrc.dec - outSrc.puts("}") - } else { - outSrc.puts("break;") - outSrc.dec - } + outSrc.puts("break;") + outSrc.dec + outSrc.puts("}") } override def switchElseStart(): Unit = { - if (switchIfs) { - outSrc.puts("else {") - outSrc.inc - } else { - outSrc.puts("default:") - outSrc.inc - } + outSrc.puts("default: {") + outSrc.inc } override def switchEnd(): Unit = - if (switchIfs) { - outSrc.dec - outSrc.puts("}") - } else { - outSrc.puts("}") - } + outSrc.puts("}") + + // + + // + + override def switchIfStart(id: Identifier, on: Ast.expr, onType: DataType): Unit = { + outSrc.puts("{") + outSrc.inc + outSrc.puts(s"${kaitaiType2NativeType(onType)} on = ${expression(on)};") + } + + override def switchIfCaseFirstStart(condition: Ast.expr): Unit = { + outSrc.puts(s"if (on == ${expression(condition)}) {") + outSrc.inc + } + + override def switchIfCaseStart(condition: Ast.expr): Unit = { + outSrc.puts(s"else if (on == ${expression(condition)}) {") + outSrc.inc + } + + override def switchIfCaseEnd(): Unit = { + outSrc.dec + outSrc.puts("}") + } + + override def switchIfElseStart(): Unit = { + outSrc.puts("else {") + outSrc.inc + } + + override def switchIfEnd(): Unit = { + outSrc.dec + outSrc.puts("}") + } + + // override def switchBytesOnlyAsRaw = true - override def instanceDeclaration(attrName: InstanceIdentifier, attrType: DataType, condSpec: ConditionalSpec): Unit = { + override def instanceDeclaration(attrName: InstanceIdentifier, attrType: DataType, isNullable: Boolean): Unit = { ensureMode(PrivateAccess) outHdr.puts(s"bool ${calculatedFlagForName(attrName)};") outHdr.puts(s"${kaitaiType2NativeType(attrType)} ${privateMemberName(attrName)};") - declareNullFlag(attrName, condSpec) + declareNullFlag(attrName, isNullable) } - override def instanceHeader(className: List[String], instName: InstanceIdentifier, dataType: DataType): Unit = { + override def instanceHeader(className: List[String], instName: InstanceIdentifier, dataType: DataType, isNullable: Boolean): Unit = { ensureMode(PublicAccess) - outHdr.puts(s"${kaitaiType2NativeType(dataType)} ${publicMemberName(instName)}();") + outHdr.puts(s"${kaitaiType2NativeType(dataType.asNonOwning)} ${publicMemberName(instName)}();") outSrc.puts - outSrc.puts(s"${kaitaiType2NativeType(dataType, true)} ${types2class(className)}::${publicMemberName(instName)}() {") + outSrc.puts(s"${kaitaiType2NativeType(dataType.asNonOwning, true)} ${types2class(className)}::${publicMemberName(instName)}() {") outSrc.inc } @@ -558,18 +822,17 @@ class CppCompiler( outSrc.puts("}") } - override def instanceCheckCacheAndReturn(instName: InstanceIdentifier): Unit = { + override def instanceCheckCacheAndReturn(instName: InstanceIdentifier, dataType: DataType): Unit = { outSrc.puts(s"if (${calculatedFlagForName(instName)})") outSrc.inc - instanceReturn(instName) + instanceReturn(instName, dataType) outSrc.dec } - override def instanceReturn(instName: InstanceIdentifier): Unit = { - outSrc.puts(s"return ${privateMemberName(instName)};") - } + override def instanceReturn(instName: InstanceIdentifier, attrType: DataType): Unit = + outSrc.puts(s"return ${nonOwningPointer(instName, attrType)};") - override def enumDeclaration(curClass: List[String], enumName: String, enumColl: Seq[(Long, String)]): Unit = { + override def enumDeclaration(curClass: List[String], enumName: String, enumColl: Seq[(Long, EnumValueSpec)]): Unit = { val enumClass = types2class(List(enumName)) outHdr.puts @@ -578,12 +841,12 @@ class CppCompiler( if (enumColl.size > 1) { enumColl.dropRight(1).foreach { case (id, label) => - outHdr.puts(s"${value2Const(enumName, label)} = $id,") + outHdr.puts(s"${value2Const(enumName, label.name)} = $id,") } } enumColl.last match { case (id, label) => - outHdr.puts(s"${value2Const(enumName, label)} = $id") + outHdr.puts(s"${value2Const(enumName, label.name)} = $id") } outHdr.dec @@ -592,7 +855,126 @@ class CppCompiler( def value2Const(enumName: String, label: String) = (enumName + "_" + label).toUpperCase - def kaitaiType2NativeType(attrType: DataType, absolute: Boolean = false): String = { + def defineName(className: String) = className.toUpperCase + "_H_" + + /** + * Returns name of a member that stores "calculated flag" for a given lazy + * attribute. That is, if it's true, then calculation have already taken + * place and we need to return already calculated member in a getter, or, + * if it's false, we need to calculate / parse it first. + * @param ksName attribute ID + * @return calculated flag member name associated with it + */ + def calculatedFlagForName(ksName: Identifier) = + s"f_${idToStr(ksName)}" + + /** + * Returns name of a member that stores "null flag" for a given attribute, + * that is, if it's true, then associated attribute is null. + * @param ksName attribute ID + * @return null flag member name associated with it + */ + def nullFlagForName(ksName: Identifier) = + s"n_${idToStr(ksName)}" + + override def idToStr(id: Identifier): String = { + id match { + case RawIdentifier(inner) => s"_raw_${idToStr(inner)}" + case IoStorageIdentifier(inner) => s"_io_${idToStr(inner)}" + case si: SpecialIdentifier => si.name + case ni: NamedIdentifier => ni.name + case NumberedIdentifier(idx) => s"_${NumberedIdentifier.TEMPLATE}$idx" + case ni: InstanceIdentifier => ni.name + } + } + + override def privateMemberName(id: Identifier): String = s"m_${idToStr(id)}" + + override def publicMemberName(id: Identifier): String = idToStr(id) + + override def localTemporaryName(id: Identifier): String = s"_t_${idToStr(id)}" + + override def paramName(id: Identifier): String = s"p_${idToStr(id)}" + + def declareNullFlag(attrName: Identifier, isNullable: Boolean) = { + if (isNullable) { + outHdr.puts(s"bool ${nullFlagForName(attrName)};") + ensureMode(PublicAccess) + outHdr.puts(s"bool _is_null_${idToStr(attrName)}() { ${publicMemberName(attrName)}(); return ${nullFlagForName(attrName)}; };") + ensureMode(PrivateAccess) + } + } + + override def type2class(className: String): String = CppCompiler.type2class(className) + + def kaitaiType2NativeType(attrType: DataType, absolute: Boolean = false): String = + CppCompiler.kaitaiType2NativeType(config.cppConfig, attrType, absolute) + + def nullPtr: String = config.cppConfig.pointers match { + case RawPointers => "0" + case SharedPointers | UniqueAndRawPointers => "nullptr" + } + + def nonOwningPointer(attrName: Identifier, attrType: DataType): String = { + config.cppConfig.pointers match { + case RawPointers => + privateMemberName(attrName) + case UniqueAndRawPointers => + attrType match { + case st: SwitchType => + nonOwningPointer(attrName, combineSwitchType(st)) + case t: ComplexDataType => + if (t.isOwning) { + s"${privateMemberName(attrName)}.get()" + } else { + privateMemberName(attrName) + } + case _ => + privateMemberName(attrName) + } + } + } + + def stdMoveWrap(expr: String): String = config.cppConfig.pointers match { + case UniqueAndRawPointers => s"std::move($expr)" + case _ => expr + } + + override def ksErrorName(err: KSError): String = err match { + case EndOfStreamError => "kaitai::end_of_stream_error" + case UndecidedEndiannessError => "kaitai::undecided_endianness_error" + case ValidationNotEqualError(dt) => + s"kaitai::validation_not_equal_error<${kaitaiType2NativeType(dt, true)}>" + } + + override def attrValidateExpr( + attrId: Identifier, + attrType: DataType, + checkExpr: Ast.expr, + errName: String, + errArgs: List[Ast.expr] + ): Unit = { + val errArgsStr = errArgs.map(translator.translate).mkString(", ") + importListSrc.add("kaitai/exceptions.h") + outSrc.puts(s"if (!(${translator.translate(checkExpr)})) {") + outSrc.inc + outSrc.puts(s"throw $errName($errArgsStr);") + outSrc.dec + outSrc.puts("}") + } +} + +object CppCompiler extends LanguageCompilerStatic + with StreamStructNames { + override def getCompiler( + tp: ClassTypeProvider, + config: RuntimeConfig + ): LanguageCompiler = new CppCompiler(tp, config) + + override def kstructName = "kaitai::kstruct" + override def kstreamName = "kaitai::kstream" + + def kaitaiType2NativeType(config: CppRuntimeConfig, attrType: DataType, absolute: Boolean = false): String = { attrType match { case Int1Type(false) => "uint8_t" case IntMultiType(false, Width2, _) => "uint16_t" @@ -622,7 +1004,12 @@ class CppCompiler( } else { t.name }) - s"$typeStr*" + config.pointers match { + case RawPointers => s"$typeStr*" + case SharedPointers => s"std::shared_ptr<$typeStr>" + case UniqueAndRawPointers => + if (t.isOwning) s"std::unique_ptr<$typeStr>" else s"$typeStr*" + } case t: EnumType => types2class(if (absolute) { @@ -631,74 +1018,62 @@ class CppCompiler( t.name }) - case ArrayType(inType) => s"std::vector<${kaitaiType2NativeType(inType, absolute)}>*" + case ArrayType(inType) => config.pointers match { + case RawPointers => s"std::vector<${kaitaiType2NativeType(config, inType, absolute)}>*" + case UniqueAndRawPointers => s"std::unique_ptr>" + } + case CalcArrayType(inType) => s"std::vector<${kaitaiType2NativeType(config, inType, absolute)}>*" case KaitaiStreamType => s"$kstreamName*" - case KaitaiStructType => s"$kstructName*" + case KaitaiStructType => config.pointers match { + case RawPointers => s"$kstructName*" + case SharedPointers => s"std::shared_ptr<$kstructName>" + case UniqueAndRawPointers => s"std::unique_ptr<$kstructName>" + } + case CalcKaitaiStructType => config.pointers match { + case RawPointers => s"$kstructName*" + case SharedPointers => s"std::shared_ptr<$kstructName>" + case UniqueAndRawPointers => s"$kstructName*" + } - case SwitchType(on, cases) => - kaitaiType2NativeType(TypeDetector.combineTypes( - // C++ does not have a concept of AnyType, and common use case "lots of incompatible UserTypes - // for cases + 1 BytesType for else" combined would result in exactly AnyType - so we try extra - // hard to avoid that here with this pre-filtering. In C++, "else" case with raw byte array would - // be available through _raw_* attribute anyway. - cases.filterNot { case (caseExpr, caseValue) => caseExpr == SwitchType.ELSE_CONST }.values - ), absolute) + case st: SwitchType => + kaitaiType2NativeType(config, combineSwitchType(st), absolute) } } - def defineName(className: String) = className.toUpperCase + "_H_" - - def calculatedFlagForName(ksName: InstanceIdentifier) = - s"f_${ksName.name}" - - def nullFlagForName(ksName: Identifier) = { - ksName match { - case NamedIdentifier(name) => s"n_$name" - case InstanceIdentifier(name) => s"n_$name" + /** + * C++ does not have a concept of AnyType, and common use case "lots of + * incompatible UserTypes for cases + 1 BytesType for else" combined would + * result in exactly AnyType - so we try extra hard to avoid that here with + * this pre-filtering. In C++, "else" case with raw byte array would + * be available through _raw_* attribute anyway. + * + * @param st switch type to combine into one overall type + * @return + */ + def combineSwitchType(st: SwitchType): DataType = { + val ct1 = TypeDetector.combineTypes( + st.cases.filterNot { + case (caseExpr, _) => caseExpr == SwitchType.ELSE_CONST + }.values + ) + if (st.isOwning) { + ct1 + } else { + ct1.asNonOwning } } - override def idToStr(id: Identifier): String = { - id match { - case RawIdentifier(inner) => s"_raw_${idToStr(inner)}" - case IoStorageIdentifier(inner) => s"_io_${idToStr(inner)}" - case si: SpecialIdentifier => si.name - case ni: NamedIdentifier => ni.name - case NumberedIdentifier(idx) => s"_${NumberedIdentifier.TEMPLATE}$idx" - case ni: InstanceIdentifier => ni.name - } + def types2class(typeName: Ast.typeId) = { + typeName.names.map(type2class).mkString( + if (typeName.absolute) "::" else "", + "::", + "" + ) } - override def privateMemberName(id: Identifier): String = s"m_${idToStr(id)}" - - override def publicMemberName(id: Identifier): String = idToStr(id) - - def declareNullFlag(attrName: Identifier, condSpec: ConditionalSpec) = { - if (condSpec.ifExpr.nonEmpty) { - outHdr.puts(s"bool ${nullFlagForName(attrName)};") - ensureMode(PublicAccess) - outHdr.puts(s"bool _is_null_${idToStr(attrName)}() { ${publicMemberName(attrName)}(); return ${nullFlagForName(attrName)}; };") - ensureMode(PrivateAccess) - } - } + def types2class(components: List[String]) = + components.map(type2class).mkString("::") def type2class(name: String) = name + "_t" } - -object CppCompiler extends LanguageCompilerStatic with StreamStructNames { - override def getCompiler( - tp: ClassTypeProvider, - config: RuntimeConfig - ): LanguageCompiler = new CppCompiler(tp, config) - - override def kstructName = "kaitai::kstruct" - override def kstreamName = "kaitai::kstream" - - def types2class(components: List[String]) = { - components.map { - case "kaitai_struct" => "kaitai::kstruct" - case s => s + "_t" - }.mkString("::") - } -} diff --git a/shared/src/main/scala/io/kaitai/struct/languages/GoCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/GoCompiler.scala new file mode 100644 index 000000000..c46127222 --- /dev/null +++ b/shared/src/main/scala/io/kaitai/struct/languages/GoCompiler.scala @@ -0,0 +1,602 @@ +package io.kaitai.struct.languages + +import io.kaitai.struct.datatype.DataType._ +import io.kaitai.struct.datatype._ +import io.kaitai.struct.exprlang.Ast +import io.kaitai.struct.format._ +import io.kaitai.struct.languages.components._ +import io.kaitai.struct.translators.{GoTranslator, ResultString, TranslatorResult} +import io.kaitai.struct.{ClassTypeProvider, RuntimeConfig, Utils} + +class GoCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) + extends LanguageCompiler(typeProvider, config) + with SingleOutputFile + with UpperCamelCaseClasses + with ObjectOrientedLanguage + with UniversalFooter + with UniversalDoc + with AllocateIOLocalVar + with GoReads { + import GoCompiler._ + + override val translator = new GoTranslator(out, typeProvider, importList) + + override def innerClasses = false + + override def universalFooter: Unit = { + out.dec + out.puts("}") + } + + override def indent: String = "\t" + override def outFileName(topClassName: String): String = + s"src/${config.goPackage}/$topClassName.go" + + override def outImports(topClass: ClassSpec): String = { + val imp = importList.toList + imp.size match { + case 0 => "" + case 1 => "import \"" + imp.head + "\"\n" + case _ => + "import (\n" + + imp.map((x) => indent + "\"" + x + "\"").mkString("", "\n", "\n") + + ")\n" + } + } + + override def fileHeader(topClassName: String): Unit = { + outHeader.puts(s"// $headerComment") + if (config.goPackage.nonEmpty) { + outHeader.puts + outHeader.puts(s"package ${config.goPackage}") + } + outHeader.puts + + importList.add("github.com/kaitai-io/kaitai_struct_go_runtime/kaitai") + + out.puts + } + + override def classHeader(name: List[String]): Unit = { + out.puts(s"type ${types2class(name)} struct {") + out.inc + } + + override def classFooter(name: List[String]): Unit = { + // TODO(jchw): where should this attribute actually be generated at? + typeProvider.nowClass.meta.endian match { + case Some(_: CalcEndian) | Some(InheritedEndian) => + out.puts(s"${idToStr(EndianIdentifier)} int") + case _ => + } + universalFooter + } + + override def classConstructorHeader(name: List[String], parentType: DataType, rootClassName: List[String], isHybrid: Boolean, params: List[ParamDefSpec]): Unit = {} + + override def classConstructorFooter: Unit = {} + + override def runRead(): Unit = { + out.puts("this.Read()") + } + + override def runReadCalc(): Unit = { + out.puts + out.puts(s"switch ${privateMemberName(EndianIdentifier)} {") + out.puts("case 0:") + out.inc + out.puts("err = this._read_be()") + out.dec + out.puts("case 1:") + out.inc + out.puts("err = this._read_le()") + out.dec + out.puts("default:") + out.inc + out.puts(s"err = ${GoCompiler.ksErrorName(UndecidedEndiannessError)}{}") + out.dec + out.puts("}") + } + + override def readHeader(endian: Option[FixedEndian], isEmpty: Boolean): Unit = { + endian match { + case None => + out.puts + out.puts( + s"func (this *${types2class(typeProvider.nowClass.name)}) Read(" + + s"io *$kstreamName, " + + s"parent ${kaitaiType2NativeType(typeProvider.nowClass.parentType)}, " + + s"root *${types2class(typeProvider.topClass.name)}) (err error) {" + ) + out.inc + out.puts(s"${privateMemberName(IoIdentifier)} = io") + out.puts(s"${privateMemberName(ParentIdentifier)} = parent") + out.puts(s"${privateMemberName(RootIdentifier)} = root") + typeProvider.nowClass.meta.endian match { + case Some(_: CalcEndian) => + out.puts(s"${privateMemberName(EndianIdentifier)} = -1") + case Some(InheritedEndian) => + out.puts(s"${privateMemberName(EndianIdentifier)} = " + + s"${privateMemberName(ParentIdentifier)}." + + s"${idToStr(EndianIdentifier)}") + case _ => + } + out.puts + case Some(e) => + out.puts + out.puts( + s"func (this *${types2class(typeProvider.nowClass.name)}) " + + s"_read_${e.toSuffix}() (err error) {") + out.inc + } + + } + override def readFooter(): Unit = { + out.puts("return err") + universalFooter + } + + override def attributeDeclaration(attrName: Identifier, attrType: DataType, isNullable: Boolean): Unit = { + out.puts(s"${idToStr(attrName)} ${kaitaiType2NativeType(attrType)}") + translator.returnRes = None + } + + override def attributeReader(attrName: Identifier, attrType: DataType, isNullable: Boolean): Unit = {} + + override def universalDoc(doc: DocSpec): Unit = { + out.puts + out.puts( "/**") + + doc.summary.foreach(summary => out.putsLines(" * ", summary)) + + doc.ref.foreach { + case TextRef(text) => + out.putsLines(" * ", "@see \"" + text + "\"") + case ref: UrlRef => + out.putsLines(" * ", s"@see ${ref.toAhref}") + } + + out.puts( " */") + } + + override def attrParseHybrid(leProc: () => Unit, beProc: () => Unit): Unit = { + out.puts(s"switch ${privateMemberName(EndianIdentifier)} {") + out.puts("case 0:") + out.inc + beProc() + out.dec + out.puts("case 1:") + out.inc + leProc() + out.dec + out.puts("default:") + out.inc + out.puts(s"err = ${GoCompiler.ksErrorName(UndecidedEndiannessError)}{}") + out.dec + out.puts("}") + } + + override def attrFixedContentsParse(attrName: Identifier, contents: Array[Byte]): Unit = { + out.puts(s"${privateMemberName(attrName)}, err = $normalIO.ReadBytes(${contents.length})") + + out.puts(s"if err != nil {") + out.inc + out.puts("return err") + out.dec + out.puts("}") + + importList.add("bytes") + importList.add("errors") + val expected = translator.resToStr(translator.doByteArrayLiteral(contents)) + out.puts(s"if !bytes.Equal(${privateMemberName(attrName)}, $expected) {") + out.inc + out.puts("return errors.New(\"Unexpected fixed contents\")") + out.dec + out.puts("}") + } + + override def attrProcess(proc: ProcessExpr, varSrc: Identifier, varDest: Identifier): Unit = { + val srcName = privateMemberName(varSrc) + val destName = privateMemberName(varDest) + + proc match { + case ProcessXor(xorValue) => + translator.detectType(xorValue) match { + case _: IntType => + out.puts(s"$destName = kaitai.ProcessXOR($srcName, []byte{${expression(xorValue)}})") + case _: BytesType => + out.puts(s"$destName = kaitai.ProcessXOR($srcName, ${expression(xorValue)})") + } + case ProcessZlib => + out.puts(s"$destName, err = kaitai.ProcessZlib($srcName)") + translator.outAddErrCheck() + case ProcessRotate(isLeft, rotValue) => + val expr = if (isLeft) { + expression(rotValue) + } else { + s"8 - (${expression(rotValue)})" + } + out.puts(s"$destName = kaitai.ProcessRotateLeft($srcName, int($expr))") + case ProcessCustom(name, args) => + // TODO(jchw): This hack is necessary because Go tests fail catastrophically otherwise... + out.puts(s"$destName = $srcName") + } + } + + override def allocateIO(varName: Identifier, rep: RepeatSpec): String = { + val javaName = privateMemberName(varName) + + val ioName = idToStr(IoStorageIdentifier(varName)) + + val args = rep match { + case RepeatEos | RepeatExpr(_) => s"$javaName[len($javaName) - 1]" + case RepeatUntil(_) => translator.specialName(Identifier.ITERATOR2) + case NoRepeat => javaName + } + + importList.add("bytes") + + out.puts(s"$ioName := kaitai.NewStream(bytes.NewReader($args))") + ioName + } + + override def useIO(ioEx: Ast.expr): String = { + out.puts(s"thisIo := ${expression(ioEx)}") + "thisIo" + } + + override def pushPos(io: String): Unit = { + out.puts(s"_pos, err := $io.Pos()") + translator.outAddErrCheck() + } + + override def seek(io: String, pos: Ast.expr): Unit = { + importList.add("io") + + out.puts(s"_, err = $io.Seek(int64(${expression(pos)}), io.SeekStart)") + translator.outAddErrCheck() + } + + override def popPos(io: String): Unit = { + importList.add("io") + + out.puts(s"_, err = $io.Seek(_pos, io.SeekStart)") + translator.outAddErrCheck() + } + + override def alignToByte(io: String): Unit = + out.puts(s"$io.AlignToByte()") + + override def condIfHeader(expr: Ast.expr): Unit = { + out.puts(s"if (${expression(expr)}) {") + out.inc + } + + override def condRepeatEosHeader(id: Identifier, io: String, dataType: DataType, needRaw: Boolean): Unit = { + if (needRaw) + out.puts(s"${privateMemberName(RawIdentifier(id))} = make([][]byte, 0);") + //out.puts(s"${privateMemberName(id)} = make(${kaitaiType2NativeType(ArrayType(dataType))})") + out.puts(s"for {") + out.inc + + val eofVar = translator.allocateLocalVar() + out.puts(s"${translator.localVarName(eofVar)}, err := this._io.EOF()") + translator.outAddErrCheck() + out.puts(s"if ${translator.localVarName(eofVar)} {") + out.inc + out.puts("break") + out.dec + out.puts("}") + } + + override def handleAssignmentRepeatEos(id: Identifier, r: TranslatorResult): Unit = { + val name = privateMemberName(id) + val expr = translator.resToStr(r) + out.puts(s"$name = append($name, $expr)") + } + + override def condRepeatExprHeader(id: Identifier, io: String, dataType: DataType, needRaw: Boolean, repeatExpr: Ast.expr): Unit = { + if (needRaw) + out.puts(s"${privateMemberName(RawIdentifier(id))} = make([][]byte, ${expression(repeatExpr)})") + out.puts(s"${privateMemberName(id)} = make(${kaitaiType2NativeType(ArrayType(dataType))}, ${expression(repeatExpr)})") + out.puts(s"for i := range ${privateMemberName(id)} {") + out.inc + } + + override def handleAssignmentRepeatExpr(id: Identifier, r: TranslatorResult): Unit = { + val name = privateMemberName(id) + val expr = translator.resToStr(r) + out.puts(s"$name[i] = $expr") + } + + override def condRepeatUntilHeader(id: Identifier, io: String, dataType: DataType, needRaw: Boolean, untilExpr: Ast.expr): Unit = { + if (needRaw) + out.puts(s"${privateMemberName(RawIdentifier(id))} = make([][]byte, 0);") + out.puts("for {") + out.inc + } + + override def handleAssignmentRepeatUntil(id: Identifier, r: TranslatorResult, isRaw: Boolean): Unit = { + val expr = translator.resToStr(r) + val tempVar = translator.specialName(Identifier.ITERATOR) + out.puts(s"$tempVar := $expr") + out.puts(s"${privateMemberName(id)} = append(${privateMemberName(id)}, $tempVar)") + } + + override def condRepeatUntilFooter(id: Identifier, io: String, dataType: DataType, needRaw: Boolean, untilExpr: Ast.expr): Unit = { + typeProvider._currentIteratorType = Some(dataType) + out.puts(s"if ${expression(untilExpr)} {") + out.inc + out.puts("break") + out.dec + out.puts("}") + out.dec + out.puts("}") + } + + private def castToType(r: TranslatorResult, dataType: DataType): TranslatorResult = { + dataType match { + case t @ (_: IntMultiType | _: FloatMultiType) => + ResultString(s"${kaitaiType2NativeType(t)}(${translator.resToStr(r)})") + case _ => + r + } + } + + private def combinedType(dataType: DataType) = { + dataType match { + case st: SwitchType => st.combinedType + case _ => dataType + } + } + + private def handleCompositeTypeCast(id: Identifier, r: TranslatorResult): TranslatorResult = { + id match { + case NamedIdentifier(name) => + castToType(r, combinedType(typeProvider.determineType(name))) + case _ => + r + } + } + + override def handleAssignmentSimple(id: Identifier, r: TranslatorResult): Unit = { + val expr = translator.resToStr(handleCompositeTypeCast(id, r)) + out.puts(s"${privateMemberName(id)} = $expr") + } + + override def parseExpr(dataType: DataType, io: String, defEndian: Option[FixedEndian]): String = { + dataType match { + case t: ReadableType => + s"$io.Read${Utils.capitalize(t.apiCall(defEndian))}()" + case blt: BytesLimitType => + s"$io.ReadBytes(int(${expression(blt.size)}))" + case _: BytesEosType => + s"$io.ReadBytesFull()" + case BytesTerminatedType(terminator, include, consume, eosError, _) => + s"$io.ReadBytesTerm($terminator, $include, $consume, $eosError)" + case BitsType1 => + s"$io.ReadBitsInt(1)" + case BitsType(width: Int) => + s"$io.ReadBitsInt($width)" + case t: UserType => + val addArgs = if (t.isOpaque) { + "" + } else { + val parent = t.forcedParent match { + case Some(USER_TYPE_NO_PARENT) => "null" + case Some(fp) => translator.translate(fp) + case None => "this" + } + s", $parent, _root" + } + s"${types2class(t.name)}($io$addArgs)" + } + } + +// override def bytesPadTermExpr(expr0: String, padRight: Option[Int], terminator: Option[Int], include: Boolean) = { +// val expr1 = padRight match { +// case Some(padByte) => s"$kstreamName.bytesStripRight($expr0, (byte) $padByte)" +// case None => expr0 +// } +// val expr2 = terminator match { +// case Some(term) => s"$kstreamName.bytesTerminate($expr1, (byte) $term, $include)" +// case None => expr1 +// } +// expr2 +// } + + override def switchStart(id: Identifier, on: Ast.expr): Unit = { + out.puts(s"switch (${expression(on)}) {") + } + + override def switchCaseStart(condition: Ast.expr): Unit = { + out.puts(s"case ${expression(condition)}:") + out.inc + } + + override def switchCaseEnd(): Unit = { + out.dec + } + + override def switchElseStart(): Unit = { + out.puts("default:") + out.inc + } + + override def switchEnd(): Unit = + out.puts("}") + + override def switchShouldUseCompareFn(onType: DataType): Option[String] = { + onType match { + case _: BytesType => + importList.add("bytes") + Some("bytes.Equal") + case _ => + None + } + } + + override def switchCaseStartCompareFn(compareFn: String, switchOn: Ast.expr, condition: Ast.expr): Unit = { + out.puts(s"case ${compareFn}(${expression(switchOn)}, ${expression(condition)}):") + out.inc + } + + override def instanceDeclaration(attrName: InstanceIdentifier, attrType: DataType, isNullable: Boolean): Unit = { + out.puts(s"${calculatedFlagForName(attrName)} bool") + out.puts(s"${idToStr(attrName)} ${kaitaiType2NativeType(attrType)}") + } + + override def instanceHeader(className: List[String], instName: InstanceIdentifier, dataType: DataType, isNullable: Boolean): Unit = { + out.puts(s"func (this *${types2class(className)}) ${publicMemberName(instName)}() (v ${kaitaiType2NativeType(dataType)}, err error) {") + out.inc + translator.returnRes = Some(dataType match { + case _: IntType => "0" + case _: BooleanType => "false" + case _: StrType => "\"\"" + case _ => "nil" + }) + } + + override def instanceCalculate(instName: Identifier, dataType: DataType, value: Ast.expr): Unit = { + val r = translator.translate(value) + val converted = dataType match { + case _: UserType => r + case _ => s"${kaitaiType2NativeType(dataType)}($r)" + } + out.puts(s"${privateMemberName(instName)} = $converted") + } + + override def instanceCheckCacheAndReturn(instName: InstanceIdentifier, dataType: DataType): Unit = { + out.puts(s"if (this.${calculatedFlagForName(instName)}) {") + out.inc + instanceReturn(instName, dataType) + universalFooter + } + + override def instanceReturn(instName: InstanceIdentifier, attrType: DataType): Unit = { + out.puts(s"return ${privateMemberName(instName)}, nil") + } + + override def instanceSetCalculated(instName: InstanceIdentifier): Unit = + out.puts(s"this.${calculatedFlagForName(instName)} = true") + + override def enumDeclaration(curClass: List[String], enumName: String, enumColl: Seq[(Long, EnumValueSpec)]): Unit = { + val fullEnumName: List[String] = curClass ++ List(enumName) + val fullEnumNameStr = types2class(fullEnumName) + + out.puts + out.puts(s"type $fullEnumNameStr int") + out.puts("const (") + out.inc + + enumColl.foreach { case (id, label) => + out.puts(s"${enumToStr(fullEnumName, label.name)} $fullEnumNameStr = $id") + } + + out.dec + out.puts(")") + } + + def idToStr(id: Identifier): String = { + id match { + case SpecialIdentifier(name) => name + case NamedIdentifier(name) => Utils.upperCamelCase(name) + case NumberedIdentifier(idx) => s"_${NumberedIdentifier.TEMPLATE}$idx" + case InstanceIdentifier(name) => Utils.lowerCamelCase(name) + case RawIdentifier(innerId) => "_raw_" + idToStr(innerId) + case IoStorageIdentifier(innerId) => "_io_" + idToStr(innerId) + } + } + + override def privateMemberName(id: Identifier): String = s"this.${idToStr(id)}" + + override def publicMemberName(id: Identifier): String = { + id match { + case IoIdentifier => "_IO" + case RootIdentifier => "_Root" + case ParentIdentifier => "_Parent" + case NamedIdentifier(name) => Utils.upperCamelCase(name) + case NumberedIdentifier(idx) => s"_${NumberedIdentifier.TEMPLATE}$idx" + case InstanceIdentifier(name) => Utils.upperCamelCase(name) + case RawIdentifier(innerId) => "_raw_" + idToStr(innerId) + } + } + + override def localTemporaryName(id: Identifier): String = s"_t_${idToStr(id)}" + + def calculatedFlagForName(id: Identifier) = s"_f_${idToStr(id)}" + + override def ksErrorName(err: KSError): String = GoCompiler.ksErrorName(err) +} + +object GoCompiler extends LanguageCompilerStatic + with UpperCamelCaseClasses + with StreamStructNames + with ExceptionNames { + + override def getCompiler( + tp: ClassTypeProvider, + config: RuntimeConfig + ): LanguageCompiler = new GoCompiler(tp, config) + + /** + * Determine Go data type corresponding to a KS data type. + * + * @param attrType KS data type + * @return Go data type + */ + def kaitaiType2NativeType(attrType: DataType): String = { + attrType match { + case Int1Type(false) => "uint8" + case IntMultiType(false, Width2, _) => "uint16" + case IntMultiType(false, Width4, _) => "uint32" + case IntMultiType(false, Width8, _) => "uint64" + + case Int1Type(true) => "int8" + case IntMultiType(true, Width2, _) => "int16" + case IntMultiType(true, Width4, _) => "int32" + case IntMultiType(true, Width8, _) => "int64" + + case FloatMultiType(Width4, _) => "float32" + case FloatMultiType(Width8, _) => "float64" + + case BitsType(_) => "uint64" + + case _: BooleanType => "bool" + case CalcIntType => "int" + case CalcFloatType => "float64" + + case _: StrType => "string" + case _: BytesType => "[]byte" + + case AnyType => "interface{}" + case KaitaiStreamType => "*" + kstreamName + case KaitaiStructType | CalcKaitaiStructType => kstructName + + case t: UserType => "*" + types2class(t.classSpec match { + case Some(cs) => cs.name + case None => t.name + }) + case t: EnumType => types2class(t.enumSpec.get.name) + + case ArrayType(inType) => s"[]${kaitaiType2NativeType(inType)}" + + case st: SwitchType => kaitaiType2NativeType(st.combinedType) + } + } + + def types2class(names: List[String]): String = names.map(x => type2class(x)).mkString("_") + + def enumToStr(enumTypeAbs: List[String]): String = { + val enumName = enumTypeAbs.last + val enumClass: List[String] = enumTypeAbs.dropRight(1) + enumToStr(enumClass, enumName) + } + + def enumToStr(typeName: List[String], enumName: String): String = + types2class(typeName) + "__" + type2class(enumName) + + override def kstreamName: String = "kaitai.Stream" + override def kstructName: String = "interface{}" + override def ksErrorName(err: KSError): String = s"kaitai.${err.name}" +} diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala index f5e8eee19..ad9112c50 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala @@ -1,15 +1,15 @@ package io.kaitai.struct.languages import io.kaitai.struct._ -import io.kaitai.struct.datatype.DataType import io.kaitai.struct.datatype.DataType._ +import io.kaitai.struct.datatype.{CalcEndian, DataType, FixedEndian, InheritedEndian, KSError} import io.kaitai.struct.exprlang.Ast import io.kaitai.struct.exprlang.Ast.expr import io.kaitai.struct.format._ import io.kaitai.struct.languages.components._ -import io.kaitai.struct.translators.{JavaTranslator, TypeDetector, TypeProvider} +import io.kaitai.struct.translators.JavaTranslator -class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) +class JavaCompiler(val typeProvider: ClassTypeProvider, config: RuntimeConfig) extends LanguageCompiler(typeProvider, config) with SingleOutputFile with UpperCamelCaseClasses @@ -21,10 +21,22 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) with UniversalDoc with AllocateIOLocalVar with FixedContentsUsingArrayByteLiteral + with SwitchIfOps with NoNeedForFullClassPath { import JavaCompiler._ - val translator = new JavaTranslator(typeProvider, config) + val translator = new JavaTranslator(typeProvider, importList) + + // Preprocess fromFileClass and make import + val fromFileClass = { + val pos = config.javaFromFileClass.lastIndexOf('.') + if (pos < 0) { + config.javaFromFileClass + } else { + importList.add(config.javaFromFileClass) + config.javaFromFileClass.substring(pos + 1) + } + } override def universalFooter: Unit = { out.dec @@ -35,22 +47,20 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def outFileName(topClassName: String): String = s"src/${config.javaPackage.replace('.', '/')}/${type2class(topClassName)}.java" + override def outImports(topClass: ClassSpec) = + "\n" + importList.toList.map((x) => s"import $x;").mkString("\n") + "\n" + override def fileHeader(topClassName: String): Unit = { - out.puts(s"// $headerComment") + outHeader.puts(s"// $headerComment") if (!config.javaPackage.isEmpty) { - out.puts - out.puts(s"package ${config.javaPackage};") + outHeader.puts + outHeader.puts(s"package ${config.javaPackage};") } - out.puts - out.puts("import io.kaitai.struct.*;") - out.puts - out.puts("import java.io.IOException;") - out.puts("import java.util.Arrays;") - out.puts("import java.util.ArrayList;") - out.puts("import java.util.Collections;") - out.puts("import java.util.HashMap;") - out.puts("import java.util.Map;") - out.puts("import java.nio.charset.Charset;") + + // Used in every class + importList.add(s"io.kaitai.struct.$kstructName") + importList.add(s"io.kaitai.struct.$kstreamName") + importList.add("java.io.IOException") out.puts } @@ -68,84 +78,173 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) ) out.inc - if (debug) { + if (config.readStoresPos) { out.puts("public Map _attrStart = new HashMap();") out.puts("public Map _attrEnd = new HashMap();") out.puts("public Map> _arrStart = new HashMap>();") out.puts("public Map> _arrEnd = new HashMap>();") out.puts + + importList.add("java.util.ArrayList") + importList.add("java.util.HashMap") + importList.add("java.util.Map") } - out.puts(s"public static ${type2class(name)} fromFile(String fileName) throws IOException {") - out.inc - out.puts(s"return new ${type2class(name)}(new $kstreamName(fileName));") - out.dec - out.puts("}") + val isInheritedEndian = typeProvider.nowClass.meta.endian match { + case Some(InheritedEndian) => true + case _ => false + } + + // fromFile helper makes no sense for inherited endianness structures: + // they require endianness to be parsed anyway + if (!isInheritedEndian && !config.javaFromFileClass.isEmpty && typeProvider.nowClass.params.isEmpty) { + out.puts(s"public static ${type2class(name)} fromFile(String fileName) throws IOException {") + out.inc + out.puts(s"return new ${type2class(name)}(new $fromFileClass(fileName));") + out.dec + out.puts("}") + } } - override def classConstructorHeader(name: String, parentClassName: String, rootClassName: String): Unit = { - if (config.readWrite) { - out.puts(s"public ${type2class(name)}() {") + override def classConstructorHeader(name: String, parentType: DataType, rootClassName: String, isHybrid: Boolean, params: List[ParamDefSpec]): Unit = { + typeProvider.nowClass.meta.endian match { + case Some(_: CalcEndian) | Some(InheritedEndian) => + out.puts("private Boolean _is_le;") + case _ => + // no _is_le variable + } + + val paramsArg = Utils.join(params.map((p) => + s"${kaitaiType2JavaType(p.dataType)} ${paramName(p.id)}" + ), ", ", ", ", "") + + if (isHybrid) { + // Inherited endian classes can be only internal, so they have mandatory 4th argument + // and 1..3-argument constructors don't make sense + + out.puts + out.puts(s"public ${type2class(name)}($kstreamName _io, ${kaitaiType2JavaType(parentType)} _parent, ${type2class(rootClassName)} _root, boolean _is_le$paramsArg) {") + out.inc + out.puts("super(_io);") + out.puts("this._parent = _parent;") + out.puts("this._root = _root;") + out.puts("this._is_le = _is_le;") + } else { + // Normal 3 constructors, chained into the last + + val paramsRelay = Utils.join(params.map((p) => paramName(p.id)), ", ", ", ", "") + + if (config.readWrite) { + out.puts(s"public ${type2class(name)}(${paramsArg.stripPrefix(", ")}) {") + out.inc + out.puts(s"this(null, null, null$paramsRelay);") + out.dec + out.puts("}") + } + + out.puts + out.puts(s"public ${type2class(name)}($kstreamName _io$paramsArg) {") out.inc - out.puts("this(null);") + out.puts(s"this(_io, null, null$paramsRelay);") out.dec out.puts("}") + + out.puts + out.puts(s"public ${type2class(name)}($kstreamName _io, ${kaitaiType2JavaType(parentType)} _parent$paramsArg) {") + out.inc + out.puts(s"this(_io, _parent, null$paramsRelay);") + out.dec + out.puts("}") + + out.puts + out.puts(s"public ${type2class(name)}($kstreamName _io, ${kaitaiType2JavaType(parentType)} _parent, ${type2class(rootClassName)} _root$paramsArg) {") + out.inc + out.puts("super(_io);") + out.puts("this._parent = _parent;") + if (name == rootClassName) { + out.puts("this._root = _root == null ? this : _root;") + } else { + out.puts("this._root = _root;") + } } + // Store parameters passed to us + params.foreach((p) => handleAssignmentSimple(p.id, paramName(p.id))) + } + + override def runRead(): Unit = + out.puts("_read();") + + override def runReadCalc(): Unit = { out.puts - out.puts(s"public ${type2class(name)}($kstreamName _io) {") + out.puts("if (_is_le == null) {") out.inc - out.puts("super(_io);") - if (name == rootClassName) - out.puts("this._root = this;") - if (!(debug || config.readWrite)) - out.puts("_read();") + out.puts(s"throw new $kstreamName.UndecidedEndiannessError();") out.dec - out.puts("}") - - out.puts - out.puts(s"public ${type2class(name)}($kstreamName _io, ${type2class(parentClassName)} _parent) {") - out.inc - out.puts("super(_io);") - out.puts("this._parent = _parent;") - if (name == rootClassName) - out.puts("this._root = this;") - if (!(debug || config.readWrite)) - out.puts("_read();") + out.puts("} else if (_is_le) {") + out.inc + out.puts("_readLE();") + out.dec + out.puts("} else {") + out.inc + out.puts("_readBE();") out.dec out.puts("}") + } + override def runWriteCalc(): Unit = { out.puts - out.puts(s"public ${type2class(name)}($kstreamName _io, ${type2class(parentClassName)} _parent, ${type2class(rootClassName)} _root) {") + out.puts("if (_is_le == null) {") + out.inc + out.puts(s"throw new $kstreamName.UndecidedEndiannessError();") + out.dec + out.puts("} else if (_is_le) {") out.inc - out.puts("super(_io);") - out.puts("this._parent = _parent;") - out.puts("this._root = _root;") - if (!(debug || config.readWrite)) - out.puts("_read();") + out.puts("_writeLE();") + out.dec + out.puts("} else {") + out.inc + out.puts("_writeBE();") out.dec out.puts("}") + } - out.puts - val readAccessAndType = if (debug || config.readWrite) { + override def readHeader(endian: Option[FixedEndian], isEmpty: Boolean) = { + val readAccessAndType = if (!config.autoRead) { "public" } else { "private" } - out.puts(s"$readAccessAndType void _read() {") + val suffix = endian match { + case Some(e) => s"${e.toSuffix.toUpperCase}" + case None => "" + } + out.puts(s"$readAccessAndType void _read$suffix() {") out.inc } - override def classConstructorFooter: Unit = { - universalFooter + override def writeHeader(endian: Option[FixedEndian]): Unit = { + val suffix = endian match { + case Some(e) => s"${e.toSuffix.toUpperCase}" + case None => "" + } + out.puts + out.puts(s"public void _write$suffix() {") + out.inc } - override def attributeDeclaration(attrName: Identifier, attrType: DataType, condSpec: ConditionalSpec): Unit = { - out.puts(s"private ${kaitaiType2JavaType(attrType, condSpec)} ${idToStr(attrName)};") + override def checkHeader(): Unit = { + out.puts + out.puts("public void _check() {") + out.inc } - override def attributeReader(attrName: Identifier, attrType: DataType, condSpec: ConditionalSpec): Unit = { - val javaType = kaitaiType2JavaType(attrType, condSpec) + override def attributeDeclaration(attrName: Identifier, attrType: DataType, isNullable: Boolean): Unit = { + out.puts(s"private ${kaitaiType2JavaType(attrType, isNullable)} ${idToStr(attrName)};") + } + + override def attributeReader(attrName: Identifier, attrType: DataType, isNullable: Boolean): Unit = { + val javaType = kaitaiType2JavaType(attrType, isNullable) val name = idToStr(attrName) out.puts(s"public $javaType $name() { return $name; }") @@ -159,30 +258,28 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts out.puts( "/**") - doc.summary.foreach((summary) => out.putsLines(" * ", summary)) + doc.summary.foreach(summary => out.putsLines(" * ", summary)) - doc.ref match { + doc.ref.foreach { case TextRef(text) => out.putsLines(" * ", "@see \"" + text + "\"") case ref: UrlRef => out.putsLines(" * ", s"@see ${ref.toAhref}") - case NoRef => - // no reference => output nothing } out.puts( " */") } - override def funcWriteHeader(curClass: ClassSpec): Unit = { - out.puts - out.puts("public void _write() {") + override def attrParseHybrid(leProc: () => Unit, beProc: () => Unit): Unit = { + out.puts("if (_is_le) {") out.inc - } - - override def funcCheckHeader(curClass: ClassSpec): Unit = { - out.puts - out.puts("public void _check() {") + leProc() + out.dec + out.puts("} else {") out.inc + beProc() + out.dec + out.puts("}") } override def attrFixedContentsParse(attrName: Identifier, contents: String): Unit = { @@ -205,6 +302,14 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) s"8 - (${expression(rotValue)})" } out.puts(s"$destName = $kstreamName.processRotateLeft($srcName, $expr, 1);") + case ProcessCustom(name, args) => + val namespace = name.init.mkString(".") + val procClass = namespace + + (if (namespace.nonEmpty) "." else "") + + type2class(name.last) + val procName = s"_process_${idToStr(varSrc)}" + out.puts(s"$procClass $procName = new $procClass(${args.map(expression).mkString(", ")});") + out.puts(s"$destName = $procName.decode($srcName);") } } @@ -238,7 +343,8 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) case NoRepeat => javaName } - out.puts(s"$kstreamName $ioName = new $kstreamName($args);") + importList.add("io.kaitai.struct.ByteBufferKaitaiStream") + out.puts(s"$kstreamName $ioName = new ByteBufferKaitaiStream($args);") ioName } @@ -322,12 +428,17 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) if (needRaw) out.puts(s"${privateMemberName(RawIdentifier(id))} = new ArrayList();") out.puts(s"${privateMemberName(id)} = new ${kaitaiType2JavaType(ArrayType(dataType))}();") + out.puts("{") + out.inc + out.puts("int i = 0;") out.puts(s"while (!$io.isEof()) {") out.inc + + importList.add("java.util.ArrayList") } override def condRepeatEosHeader2(id: Identifier, io: String, dataType: DataType, needRaw: Boolean): Unit = { - out.puts(s"for (int _i = 0; _i < ${privateMemberName(id)}.size(); _i++) {") + out.puts(s"for (int i = 0; i < ${privateMemberName(id)}.size(); i++) {") out.inc } @@ -335,15 +446,23 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts(s"${privateMemberName(id)}.add($expr);") } + override def condRepeatEosFooter: Unit = { + out.puts("i++;") + out.dec + out.puts("}") + } + override def condRepeatExprHeader(id: Identifier, io: String, dataType: DataType, needRaw: Boolean, repeatExpr: expr): Unit = { if (needRaw) - out.puts(s"${privateMemberName(RawIdentifier(id))} = new ArrayList((int) (${expression(repeatExpr)}));") - out.puts(s"${idToStr(id)} = new ${kaitaiType2JavaType(ArrayType(dataType))}((int) (${expression(repeatExpr)}));") + out.puts(s"${privateMemberName(RawIdentifier(id))} = new ArrayList(((Number) (${expression(repeatExpr)})).intValue());") + out.puts(s"${idToStr(id)} = new ${kaitaiType2JavaType(ArrayType(dataType))}(((Number) (${expression(repeatExpr)})).intValue());") + + importList.add("java.util.ArrayList") condRepeatExprHeader2(id, io, dataType, needRaw, repeatExpr) } override def condRepeatExprHeader2(id: Identifier, io: String, dataType: DataType, needRaw: Boolean, repeatExpr: expr): Unit = { - out.puts(s"for (int _i = 0; _i < ${expression(repeatExpr)}; _i++) {") + out.puts(s"for (int i = 0; i < ${expression(repeatExpr)}; i++) {") out.inc } @@ -358,8 +477,11 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts("{") out.inc out.puts(s"${kaitaiType2JavaType(dataType)} ${translator.doName("_")};") + out.puts("int i = 0;") out.puts("do {") out.inc + + importList.add("java.util.ArrayList") } override def handleAssignmentRepeatUntil(id: Identifier, expr: String, isRaw: Boolean): Unit = { @@ -374,6 +496,7 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def condRepeatUntilFooter(id: Identifier, io: String, dataType: DataType, needRaw: Boolean, untilExpr: expr): Unit = { typeProvider._currentIteratorType = Some(dataType) + out.puts("i++;") out.dec out.puts(s"} while (!(${expression(untilExpr)}));") out.dec @@ -386,10 +509,10 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def handleAssignmentTempVar(dataType: DataType, id: String, expr: String): Unit = out.puts(s"${kaitaiType2JavaType(dataType)} $id = $expr;") - override def parseExpr(dataType: DataType, io: String): String = { - dataType match { + override def parseExpr(dataType: DataType, assignType: DataType, io: String, defEndian: Option[FixedEndian]): String = { + val expr = dataType match { case t: ReadableType => - s"$io.read${Utils.capitalize(t.apiCall)}()" + s"$io.read${Utils.capitalize(t.apiCall(defEndian))}()" case blt: BytesLimitType => s"$io.readBytes(${expression(blt.size)})" case _: BytesEosType => @@ -409,9 +532,20 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) case Some(fp) => translator.translate(fp) case None => "this" } - s", $parent, _root" + val addEndian = t.classSpec.get.meta.endian match { + case Some(InheritedEndian) => ", _is_le" + case _ => "" + } + s", $parent, _root$addEndian" } - s"new ${types2class(t.name)}($io$addArgs)" + val addParams = Utils.join(t.args.map((a) => translator.translate(a)), ", ", ", ", "") + s"new ${types2class(t.name)}($io$addArgs$addParams)" + } + + if (assignType != dataType) { + s"(${kaitaiType2JavaType(assignType)}) ($expr)" + } else { + expr } } @@ -430,17 +564,74 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def userTypeDebugRead(id: String): Unit = out.puts(s"$id._read();") + override def switchCasesRender[T]( + id: Identifier, + on: Ast.expr, + cases: Map[Ast.expr, T], + normalCaseProc: T => Unit, + elseCaseProc: T => Unit + ): Unit = { + // Java has a stupid limitation of being unable to match nulls in switch. + // If our type is nullable, we'll do an extra check. For now, we're only + // doing this workaround for enums. + + val onType = typeProvider._currentSwitchType.get + val isNullable = onType match { + case _: EnumType => true + case _ => false + } + + if (isNullable) { + val nameSwitchStr = expression(NAME_SWITCH_ON) + out.puts("{") + out.inc + out.puts(s"${kaitaiType2JavaType(onType)} $nameSwitchStr = ${expression(on)};") + out.puts(s"if ($nameSwitchStr != null) {") + out.inc + + super.switchCasesRender(id, on, cases, normalCaseProc, elseCaseProc) + + out.dec + cases.get(SwitchType.ELSE_CONST) match { + case Some(result) => + out.puts("} else {") + out.inc + elseCaseProc(result) + out.dec + out.puts("}") + case None => + out.puts("}") + } + + out.dec + out.puts("}") + } else { + super.switchCasesRender(id, on, cases, normalCaseProc, elseCaseProc) + } + } + + override def switchRequiresIfs(onType: DataType): Boolean = onType match { + case _: IntType | _: EnumType | _: StrType => false + case _ => true + } + + // + + val NAME_SWITCH_ON = Ast.expr.Name(Ast.identifier(Identifier.SWITCH_ON)) + override def switchStart(id: Identifier, on: Ast.expr): Unit = out.puts(s"switch (${expression(on)}) {") + override def switchCaseFirstStart(condition: Ast.expr): Unit = switchCaseStart(condition) + override def switchCaseStart(condition: Ast.expr): Unit = { // Java is very specific about what can be used as "condition" in "case // condition:". val condStr = condition match { - case Ast.expr.EnumByLabel(enumName, enumVal) => + case enumByLabel: Ast.expr.EnumByLabel => // If switch is over a enum, only literal enum values are supported, // and they must be written as "MEMBER", not "SomeEnum.MEMBER". - value2Const(enumVal.name) + value2Const(enumByLabel.label.name) case _ => expression(condition) } @@ -460,30 +651,77 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.inc } - override def switchEnd(): Unit = + override def switchEnd(): Unit = { out.puts("}") + } + + // + + // - override def instanceDeclaration(attrName: InstanceIdentifier, attrType: DataType, condSpec: ConditionalSpec): Unit = { + override def switchIfStart(id: Identifier, on: expr, onType: DataType): Unit = { + out.puts("{") + out.inc + out.puts(s"${kaitaiType2JavaType(onType)} ${expression(NAME_SWITCH_ON)} = ${expression(on)};") + } + + def switchCmpExpr(condition: Ast.expr): String = + expression( + Ast.expr.Compare( + NAME_SWITCH_ON, + Ast.cmpop.Eq, + condition + ) + ) + + override def switchIfCaseFirstStart(condition: Ast.expr): Unit = { + out.puts(s"if (${switchCmpExpr(condition)}) {") + out.inc + } + + override def switchIfCaseStart(condition: Ast.expr): Unit = { + out.puts(s"else if (${switchCmpExpr(condition)}) {") + out.inc + } + + override def switchIfCaseEnd(): Unit = { + out.dec + out.puts("}") + } + + override def switchIfElseStart(): Unit = { + out.puts("else {") + out.inc + } + + override def switchIfEnd(): Unit = { + out.dec + out.puts("}") + } + + // + + override def instanceDeclaration(attrName: InstanceIdentifier, attrType: DataType, isNullable: Boolean): Unit = { out.puts(s"private ${kaitaiType2JavaTypeBoxed(attrType)} ${idToStr(attrName)};") } - override def instanceHeader(className: String, instName: InstanceIdentifier, dataType: DataType): Unit = { + override def instanceHeader(className: String, instName: InstanceIdentifier, dataType: DataType, isNullable: Boolean): Unit = { out.puts(s"public ${kaitaiType2JavaTypeBoxed(dataType)} ${idToStr(instName)}() {") out.inc } - override def instanceCheckCacheAndReturn(instName: InstanceIdentifier): Unit = { + override def instanceCheckCacheAndReturn(instName: InstanceIdentifier, dataType: DataType): Unit = { out.puts(s"if (${privateMemberName(instName)} != null)") out.inc - instanceReturn(instName) + instanceReturn(instName, dataType) out.dec } - override def instanceReturn(instName: InstanceIdentifier): Unit = { + override def instanceReturn(instName: InstanceIdentifier, attrType: DataType): Unit = { out.puts(s"return ${privateMemberName(instName)};") } - override def instanceCalculate(instName: InstanceIdentifier, dataType: DataType, value: expr): Unit = { + override def instanceCalculate(instName: Identifier, dataType: DataType, value: expr): Unit = { val primType = kaitaiType2JavaTypePrim(dataType) val boxedType = kaitaiType2JavaTypeBoxed(dataType) @@ -533,6 +771,9 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts(s"public static $enumClass byId(long id) { return byId.get(id); }") out.dec out.puts("}") + + importList.add("java.util.Map") + importList.add("java.util.HashMap") } override def debugClassSequence(seq: List[AttrSpec]) = { @@ -540,16 +781,16 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts(s"public static String[] _seqFields = new String[] { $seqStr };") } - override def attrPrimitiveWrite(io: String, expr: String, dataType: DataType): Unit = { + override def attrPrimitiveWrite(io: String, expr: String, dataType: DataType, defEndian: Option[FixedEndian]): Unit = { val stmt = dataType match { case t: ReadableType => - s"$io.write${Utils.capitalize(t.apiCall)}($expr)" + s"$io.write${Utils.capitalize(t.apiCall(defEndian))}($expr))" case BitsType1 => - s"$io.writeBitsInt(1, ($expr) ? 1 : 0)" + s"$io.writeBitsInt(1, ($expr)}) ? 1 : 0)" case BitsType(width: Int) => - s"$io.writeBitsInt($width, $expr)" + s"$io.writeBitsInt($width, $expr)})" case _: BytesType => - s"$io.writeBytes($expr)" + s"$io.writeBytes($expr)})" } out.puts(stmt + ";") } @@ -601,10 +842,47 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def publicMemberName(id: Identifier) = idToStr(id) + override def localTemporaryName(id: Identifier): String = s"_t_${idToStr(id)}" + + override def ksErrorName(err: KSError): String = JavaCompiler.ksErrorName(err) + + override def attrValidateExpr( + attrId: Identifier, + attrType: DataType, + checkExpr: Ast.expr, + errName: String, + errArgs: List[Ast.expr] + ): Unit = { + val errArgsStr = errArgs.map(translator.translate).mkString(", ") + out.puts(s"if (!(${translator.translate(checkExpr)})) {") + out.inc + out.puts(s"throw new $errName($errArgsStr);") + out.dec + out.puts("}") + } + + def kstructNameFull: String = { + kstructName + ((config.autoRead, config.readWrite) match { + case (_, true) => ".ReadWrite" + case (false, false) => ".ReadOnly" + case (true, false) => "" + }) + } +} + +object JavaCompiler extends LanguageCompilerStatic + with UpperCamelCaseClasses + with StreamStructNames + with ExceptionNames { + override def getCompiler( + tp: ClassTypeProvider, + config: RuntimeConfig + ): LanguageCompiler = new JavaCompiler(tp, config) + def kaitaiType2JavaType(attrType: DataType): String = kaitaiType2JavaTypePrim(attrType) - def kaitaiType2JavaType(attrType: DataType, condSpec: ConditionalSpec): String = - if (condSpec.ifExpr.nonEmpty) { + def kaitaiType2JavaType(attrType: DataType, isNullable: Boolean): String = + if (isNullable) { kaitaiType2JavaTypeBoxed(attrType) } else { kaitaiType2JavaTypePrim(attrType) @@ -643,14 +921,14 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) case AnyType => "Object" case KaitaiStreamType => kstreamName - case KaitaiStructType => kstructNameFull + case KaitaiStructType | CalcKaitaiStructType => kstructName // FIXME: should be `kstructNameFull`! case t: UserType => types2class(t.name) case EnumType(name, _) => types2class(name) case ArrayType(_) => kaitaiType2JavaTypeBoxed(attrType) - case SwitchType(_, cases) => kaitaiType2JavaTypePrim(TypeDetector.combineTypes(cases.values)) + case st: SwitchType => kaitaiType2JavaTypePrim(st.combinedType) } } @@ -687,37 +965,20 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) case AnyType => "Object" case KaitaiStreamType => kstreamName - case KaitaiStructType => kstructNameFull + case KaitaiStructType | CalcKaitaiStructType => kstructName // FIXME: should be `kstructNameFull`! - case t: UserType => type2class(t.name.last) + case t: UserType => types2class(t.name) case EnumType(name, _) => types2class(name) case ArrayType(inType) => s"ArrayList<${kaitaiType2JavaTypeBoxed(inType)}>" - case SwitchType(_, cases) => kaitaiType2JavaTypeBoxed(TypeDetector.combineTypes(cases.values)) + case st: SwitchType => kaitaiType2JavaTypeBoxed(st.combinedType) } } def types2class(names: List[String]) = names.map(x => type2class(x)).mkString(".") - def kstructNameFull: String = { - kstructName + ((config.debug, config.readWrite) match { - case (_, true) => ".ReadWrite" - case (true, false) => ".ReadOnly" - case (false, false) => "" - }) - } -} - -object JavaCompiler extends LanguageCompilerStatic - with UpperCamelCaseClasses - with StreamStructNames { - - override def getCompiler( - tp: ClassTypeProvider, - config: RuntimeConfig - ): LanguageCompiler = new JavaCompiler(tp, config) - override def kstreamName: String = "KaitaiStream" override def kstructName: String = "KaitaiStruct" + override def ksErrorName(err: KSError): String = s"KaitaiStream.${err.name}" } diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JavaScriptCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JavaScriptCompiler.scala index 3168409ee..221652fe6 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/JavaScriptCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/JavaScriptCompiler.scala @@ -1,15 +1,15 @@ package io.kaitai.struct.languages -import io.kaitai.struct.datatype.DataType import io.kaitai.struct.datatype.DataType._ +import io.kaitai.struct.datatype.{DataType, FixedEndian, InheritedEndian, KSError} import io.kaitai.struct.exprlang.Ast import io.kaitai.struct.exprlang.Ast.expr import io.kaitai.struct.format._ import io.kaitai.struct.languages.components._ -import io.kaitai.struct.translators.{JavaScriptTranslator, TypeProvider} -import io.kaitai.struct.{ClassTypeProvider, LanguageOutputWriter, RuntimeConfig, Utils} +import io.kaitai.struct.translators.JavaScriptTranslator +import io.kaitai.struct.{ClassTypeProvider, RuntimeConfig, Utils} -class JavaScriptCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) +class JavaScriptCompiler(val typeProvider: ClassTypeProvider, config: RuntimeConfig) extends LanguageCompiler(typeProvider, config) with ObjectOrientedLanguage with UpperCamelCaseClasses @@ -17,6 +17,7 @@ class JavaScriptCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) with UniversalDoc with AllocateIOLocalVar with EveryReadIsExpression + with SwitchIfOps with FixedContentsUsingArrayByteLiteral { import JavaScriptCompiler._ @@ -25,40 +26,40 @@ class JavaScriptCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def indent: String = " " override def outFileName(topClassName: String): String = s"${type2class(topClassName)}.js" + override def outImports(topClass: ClassSpec) = { + val impList = importList.toList + val quotedImpList = impList.map((x) => s"'$x'") + val defineArgs = quotedImpList.mkString(", ") + val moduleArgs = quotedImpList.map((x) => s"require($x)").mkString(", ") + val argClasses = impList.map((x) => x.split('/').last) + val rootArgs = argClasses.map((x) => s"root.$x").mkString(", ") + + "(function (root, factory) {\n" + + indent + "if (typeof define === 'function' && define.amd) {\n" + + indent * 2 + s"define([$defineArgs], factory);\n" + + indent + "} else if (typeof module === 'object' && module.exports) {\n" + + indent * 2 + s"module.exports = factory($moduleArgs);\n" + + indent + "} else {\n" + + indent * 2 + s"root.${types2class(topClass.name)} = factory($rootArgs);\n" + + indent + "}\n" + + s"}(this, function (${argClasses.mkString(", ")}) {" + } + override def fileHeader(topClassName: String): Unit = { - out.puts(s"// $headerComment") + outHeader.puts(s"// $headerComment") + outHeader.puts + + importList.add("kaitai-struct/KaitaiStream") } override def fileFooter(name: String): Unit = { - out.puts - out.puts("// Export for amd environments") - out.puts("if (typeof define === 'function' && define.amd) {") - out.inc - out.puts(s"define('${type2class(name)}', [], function() {") - out.inc out.puts(s"return ${type2class(name)};") - out.dec - out.puts("});") - out.dec - out.puts("}") - - out.puts - - out.puts("// Export for CommonJS") - out.puts("if (typeof module === 'object' && module && module.exports) {") - out.inc - out.puts(s"module.exports = ${type2class(name)};") - out.dec - out.puts("}") + out.puts("}));") } override def opaqueClassDeclaration(classSpec: ClassSpec): Unit = { - val typeName = classSpec.name.head - out.puts - out.puts("if (typeof require === 'function')") - out.inc - out.puts(s"var ${type2class(typeName)} = require('./${outFileName(typeName)}');") - out.dec + val className = type2class(classSpec.name.head) + importList.add(s"./$className") } override def classHeader(name: List[String]): Unit = { @@ -82,22 +83,31 @@ class JavaScriptCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts("})();") } - override def classConstructorHeader(name: List[String], parentClassName: List[String], rootClassName: List[String]): Unit = { - out.puts(s"function ${type2class(name.last)}(_io, _parent, _root) {") + override def classConstructorHeader(name: List[String], parentClassName: DataType, rootClassName: List[String], isHybrid: Boolean, params: List[ParamDefSpec]): Unit = { + val endianSuffix = if (isHybrid) { + ", _is_le" + } else { + "" + } + + val paramsList = Utils.join(params.map((p) => paramName(p.id)), ", ", ", ", "") + + out.puts(s"function ${type2class(name.last)}(_io, _parent, _root$endianSuffix$paramsList) {") out.inc out.puts("this._io = _io;") out.puts("this._parent = _parent;") out.puts("this._root = _root || this;") - if (debug) { + + if (isHybrid) + out.puts("this._is_le = _is_le;") + + // Store parameters passed to us + params.foreach((p) => handleAssignmentSimple(p.id, paramName(p.id))) + + if (config.readStoresPos) { out.puts("this._debug = {};") - out.dec - out.puts("}") - out.puts - out.puts(s"${type2class(name.last)}.prototype._read = function() {") - out.inc - } else { - out.puts } + out.puts } override def classConstructorFooter: Unit = { @@ -105,29 +115,75 @@ class JavaScriptCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts("}") } - override def attributeDeclaration(attrName: Identifier, attrType: DataType, condSpec: ConditionalSpec): Unit = {} + override def runRead(): Unit = { + out.puts("this._read();") + } - override def attributeReader(attrName: Identifier, attrType: DataType, condSpec: ConditionalSpec): Unit = {} + override def runReadCalc(): Unit = { + out.puts + out.puts(s"if (this._is_le === true) {") + out.inc + out.puts("this._readLE();") + out.dec + out.puts("} else if (this._is_le === false) {") + out.inc + out.puts("this._readBE();") + out.dec + out.puts("} else {") + out.inc + out.puts("throw new KaitaiStream.UndecidedEndiannessError();") + out.dec + out.puts("}") + } + + override def readHeader(endian: Option[FixedEndian], isEmpty: Boolean) = { + val suffix = endian match { + case Some(e) => e.toSuffix.toUpperCase + case None => "" + } + out.puts(s"${type2class(typeProvider.nowClass.name.last)}.prototype._read$suffix = function() {") + out.inc + } + + override def readFooter() = { + out.dec + out.puts("}") + } + + override def attributeDeclaration(attrName: Identifier, attrType: DataType, isNullable: Boolean): Unit = {} + + override def attributeReader(attrName: Identifier, attrType: DataType, isNullable: Boolean): Unit = {} override def universalDoc(doc: DocSpec): Unit = { // JSDoc docstring style: http://usejsdoc.org/about-getting-started.html out.puts out.puts( "/**") - doc.summary.foreach((summary) => out.putsLines(" * ", summary)) + doc.summary.foreach(summary => out.putsLines(" * ", summary)) // http://usejsdoc.org/tags-see.html - doc.ref match { + doc.ref.foreach { case TextRef(text) => out.putsLines(" * ", s"@see $text") case UrlRef(url, text) => out.putsLines(" * ", s"@see {@link $url|$text}") - case NoRef => } out.puts( " */") } + override def attrParseHybrid(leProc: () => Unit, beProc: () => Unit): Unit = { + out.puts("if (this._is_le) {") + out.inc + leProc() + out.dec + out.puts("} else {") + out.inc + beProc() + out.dec + out.puts("}") + } + override def attrFixedContentsParse(attrName: Identifier, contents: String): Unit = { out.puts(s"${privateMemberName(attrName)} = " + s"$normalIO.ensureFixedContents($contents);") @@ -153,6 +209,15 @@ class JavaScriptCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) s"8 - (${expression(rotValue)})" } out.puts(s"$destName = $kstreamName.processRotateLeft($srcName, $expr, 1);") + case ProcessCustom(name, args) => + val nameInit = name.init + val pkgName = if (nameInit.isEmpty) "" else nameInit.mkString("-") + "/" + val procClass = type2class(name.last) + + importList.add(s"$pkgName$procClass") + + out.puts(s"var _process = new $procClass(${args.map(expression).mkString(", ")});") + out.puts(s"$destName = _process.decode($srcName);") } } @@ -230,8 +295,9 @@ class JavaScriptCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) if (needRaw) out.puts(s"${privateMemberName(RawIdentifier(id))} = [];") out.puts(s"${privateMemberName(id)} = [];") - if (debug) + if (config.readStoresPos) out.puts(s"this._debug.${idToStr(id)}.arr = [];") + out.puts("var i = 0;") out.puts(s"while (!$io.isEof()) {") out.inc } @@ -241,6 +307,7 @@ class JavaScriptCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } override def condRepeatEosFooter: Unit = { + out.puts("i++;") out.dec out.puts("}") } @@ -249,7 +316,7 @@ class JavaScriptCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) if (needRaw) out.puts(s"${privateMemberName(RawIdentifier(id))} = new Array(${expression(repeatExpr)});") out.puts(s"${privateMemberName(id)} = new Array(${expression(repeatExpr)});") - if (debug) + if (config.readStoresPos) out.puts(s"this._debug.${idToStr(id)}.arr = new Array(${expression(repeatExpr)});") out.puts(s"for (var i = 0; i < ${expression(repeatExpr)}; i++) {") out.inc @@ -268,8 +335,9 @@ class JavaScriptCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) if (needRaw) out.puts(s"${privateMemberName(RawIdentifier(id))} = []") out.puts(s"${privateMemberName(id)} = []") - if (debug) + if (config.readStoresPos) out.puts(s"this._debug.${idToStr(id)}.arr = [];") + out.puts("var i = 0;") out.puts("do {") out.inc } @@ -282,6 +350,7 @@ class JavaScriptCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def condRepeatUntilFooter(id: Identifier, io: String, dataType: DataType, needRaw: Boolean, untilExpr: expr): Unit = { typeProvider._currentIteratorType = Some(dataType) + out.puts("i++;") out.dec out.puts(s"} while (!(${expression(untilExpr)}));") } @@ -293,10 +362,10 @@ class JavaScriptCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def handleAssignmentTempVar(dataType: DataType, id: String, expr: String): Unit = out.puts(s"var $id = $expr;") - override def parseExpr(dataType: DataType, io: String): String = { + override def parseExpr(dataType: DataType, assignType: DataType, io: String, defEndian: Option[FixedEndian]): String = { dataType match { case t: ReadableType => - s"$io.read${Utils.capitalize(t.apiCall)}()" + s"$io.read${Utils.capitalize(t.apiCall(defEndian))}()" case blt: BytesLimitType => s"$io.readBytes(${expression(blt.size)})" case _: BytesEosType => @@ -308,8 +377,18 @@ class JavaScriptCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) case BitsType(width: Int) => s"$io.readBitsInt($width)" case t: UserType => - val addArgs = if (t.isOpaque) "" else ", this, this._root" - s"new ${type2class(t.name.last)}($io$addArgs)" + val parent = t.forcedParent match { + case Some(USER_TYPE_NO_PARENT) => "null" + case Some(fp) => translator.translate(fp) + case None => "this" + } + val root = if (t.isOpaque) "null" else "this._root" + val addEndian = t.classSpec.get.meta.endian match { + case Some(InheritedEndian) => ", this._is_le" + case _ => "" + } + val addParams = Utils.join(t.args.map((a) => translator.translate(a)), ", ", ", ", "") + s"new ${types2class(t.name)}($io, $parent, $root$addEndian$addParams)" } } @@ -329,9 +408,19 @@ class JavaScriptCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts(s"$id._read();") } + override def switchRequiresIfs(onType: DataType): Boolean = onType match { + case _: IntType | _: BooleanType | _: EnumType | _: StrType => false + case _ => true + } + + // + override def switchStart(id: Identifier, on: Ast.expr): Unit = out.puts(s"switch (${expression(on)}) {") + override def switchCaseFirstStart(condition: Ast.expr): Unit = + switchCaseStart(condition) + override def switchCaseStart(condition: Ast.expr): Unit = { out.puts(s"case ${expression(condition)}:") out.inc @@ -350,7 +439,55 @@ class JavaScriptCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def switchEnd(): Unit = out.puts("}") - override def instanceHeader(className: List[String], instName: InstanceIdentifier, dataType: DataType): Unit = { + // + + // + + val NAME_SWITCH_ON = Ast.expr.Name(Ast.identifier(Identifier.SWITCH_ON)) + + override def switchIfStart(id: Identifier, on: Ast.expr, onType: DataType): Unit = { + out.puts("{") + out.inc + out.puts(s"var ${expression(NAME_SWITCH_ON)} = ${expression(on)};") + } + + private def switchCmpExpr(condition: Ast.expr): String = + expression( + Ast.expr.Compare( + NAME_SWITCH_ON, + Ast.cmpop.Eq, + condition + ) + ) + + override def switchIfCaseFirstStart(condition: Ast.expr): Unit = { + out.puts(s"if (${switchCmpExpr(condition)}) {") + out.inc + } + + override def switchIfCaseStart(condition: Ast.expr): Unit = { + out.puts(s"else if (${switchCmpExpr(condition)}) {") + out.inc + } + + override def switchIfCaseEnd(): Unit = { + out.dec + out.puts("}") + } + + override def switchIfElseStart(): Unit = { + out.puts("else {") + out.inc + } + + override def switchIfEnd(): Unit = { + out.dec + out.puts("}") + } + + // + + override def instanceHeader(className: List[String], instName: InstanceIdentifier, dataType: DataType, isNullable: Boolean): Unit = { out.puts(s"Object.defineProperty(${type2class(className.last)}.prototype, '${publicMemberName(instName)}', {") out.inc out.puts("get: function() {") @@ -364,27 +501,37 @@ class JavaScriptCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts("});") } - override def instanceCheckCacheAndReturn(instName: InstanceIdentifier): Unit = { + override def instanceCheckCacheAndReturn(instName: InstanceIdentifier, dataType: DataType): Unit = { out.puts(s"if (${privateMemberName(instName)} !== undefined)") out.inc - instanceReturn(instName) + instanceReturn(instName, dataType) out.dec } - override def instanceReturn(instName: InstanceIdentifier): Unit = { + override def instanceReturn(instName: InstanceIdentifier, attrType: DataType): Unit = { out.puts(s"return ${privateMemberName(instName)};") } - override def enumDeclaration(curClass: List[String], enumName: String, enumColl: Seq[(Long, String)]): Unit = { + override def enumDeclaration(curClass: List[String], enumName: String, enumColl: Seq[(Long, EnumValueSpec)]): Unit = { out.puts(s"${type2class(curClass.last)}.${type2class(enumName)} = Object.freeze({") out.inc + + // Name to ID mapping enumColl.foreach { case (id, label) => - out.puts(s"${enumValue(enumName, label)}: $id,") + out.puts(s"${enumValue(enumName, label.name)}: $id,") } out.puts + + // ID to name mapping enumColl.foreach { case (id, label) => - out.puts(s"""$id: "${enumValue(enumName, label)}",""") + val idStr = if (id < 0) { + "\"" + id.toString + "\"" + } else { + id.toString + } + out.puts(s"""$idStr: "${enumValue(enumName, label.name)}",""") } + out.dec out.puts("});") out.puts @@ -416,6 +563,25 @@ class JavaScriptCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } } + override def localTemporaryName(id: Identifier): String = s"_t_${idToStr(id)}" + + override def ksErrorName(err: KSError): String = JavaScriptCompiler.ksErrorName(err) + + override def attrValidateExpr( + attrId: Identifier, + attrType: DataType, + checkExpr: Ast.expr, + errName: String, + errArgs: List[Ast.expr] + ): Unit = { + val errArgsStr = errArgs.map(translator.translate).mkString(", ") + out.puts(s"if (!(${translator.translate(checkExpr)})) {") + out.inc + out.puts(s"throw new $errName($errArgsStr);") + out.dec + out.puts("}") + } + private def attrDebugNeeded(attrId: Identifier) = attrId match { case _: NamedIdentifier | _: NumberedIdentifier | _: InstanceIdentifier => true @@ -435,7 +601,8 @@ class JavaScriptCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) object JavaScriptCompiler extends LanguageCompilerStatic with UpperCamelCaseClasses - with StreamStructNames { + with StreamStructNames + with ExceptionNames { override def getCompiler( tp: ClassTypeProvider, config: RuntimeConfig @@ -446,6 +613,7 @@ object JavaScriptCompiler extends LanguageCompilerStatic // FIXME: probably KaitaiStruct will emerge some day in JavaScript runtime, but for now it is unused override def kstructName: String = ??? - def types2class(types: List[String]): String = - types.map(JavaScriptCompiler.type2class).mkString(".") + override def ksErrorName(err: KSError): String = s"KaitaiStream.${err.name}" + + def types2class(types: List[String]): String = types.map(type2class).mkString(".") } diff --git a/shared/src/main/scala/io/kaitai/struct/languages/LuaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/LuaCompiler.scala new file mode 100644 index 000000000..34d774810 --- /dev/null +++ b/shared/src/main/scala/io/kaitai/struct/languages/LuaCompiler.scala @@ -0,0 +1,412 @@ +package io.kaitai.struct.languages + +import io.kaitai.struct.{ClassTypeProvider, RuntimeConfig, Utils} +import io.kaitai.struct.datatype.{DataType, FixedEndian, InheritedEndian, KSError} +import io.kaitai.struct.datatype.DataType._ +import io.kaitai.struct.exprlang.Ast +import io.kaitai.struct.format._ +import io.kaitai.struct.languages.components._ +import io.kaitai.struct.translators.LuaTranslator + +class LuaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) + extends LanguageCompiler(typeProvider, config) + with AllocateIOLocalVar + with EveryReadIsExpression + with FixedContentsUsingArrayByteLiteral + with ObjectOrientedLanguage + with SingleOutputFile + with UniversalDoc + with UniversalFooter + with UpperCamelCaseClasses { + + import LuaCompiler._ + + override val translator = new LuaTranslator(typeProvider, importList) + + override def innerClasses = false + override def innerEnums = true + + override def indent: String = " " + override def outFileName(topClassName: String): String = s"$topClassName.lua" + override def outImports(topClass: ClassSpec) = + importList.toList.mkString("", "\n", "\n") + + override def opaqueClassDeclaration(classSpec: ClassSpec): Unit = + out.puts("require(\"" + classSpec.name.head + "\")") + + override def fileHeader(topClassName: String): Unit = { + outHeader.puts(s"-- $headerComment") + outHeader.puts("--") + outHeader.puts("-- This file is compatible with Lua 5.3") + outHeader.puts + + importList.add("local class = require(\"class\")") + importList.add("require(\"kaitaistruct\")") + + out.puts + } + + override def universalFooter: Unit = + out.puts + + override def universalDoc(doc: DocSpec): Unit = { + val docStr = doc.summary match { + case Some(summary) => + val lastChar = summary.last + if (lastChar == '.' || lastChar == '\n') { + summary + } else { + summary + "." + } + case None => + "" + } + val extraNewLine = if (docStr.isEmpty || docStr.last == '\n') "" else "\n" + val refStr = doc.ref.map { + case TextRef(text) => + s"See also: $text" + case UrlRef(url, text) => + s"See also: $text ($url)" + }.mkString("\n") + + out.putsLines("-- ", "\n" + docStr + extraNewLine + refStr) + } + + override def classHeader(name: List[String]): Unit = { + out.puts(s"${types2class(name)} = class.class($kstructName)") + out.puts + } + override def classFooter(name: List[String]): Unit = + universalFooter + override def classConstructorHeader(name: List[String], parentType: DataType, rootClassName: List[String], isHybrid: Boolean, params: List[ParamDefSpec]): Unit = { + val endianAdd = if (isHybrid) ", is_le" else "" + val paramsList = Utils.join(params.map((p) => paramName(p.id)), "", ", ", ", ") + + out.puts(s"function ${types2class(name)}:_init($paramsList" + s"io, parent, root$endianAdd)") + out.inc + out.puts(s"$kstructName._init(self, io)") + out.puts("self._parent = parent") + out.puts("self._root = root or self") + if (isHybrid) + out.puts("self._is_le = is_le") + + // Store parameters passed to us + params.foreach((p) => handleAssignmentSimple(p.id, paramName(p.id))) + } + override def classConstructorFooter: Unit = { + out.dec + out.puts("end") + out.puts + } + + override def runRead(): Unit = + out.puts("self:_read()") + override def runReadCalc(): Unit = { + out.puts + out.puts(s"if self._is_le then") + out.inc + out.puts("self:_read_le()") + out.dec + out.puts(s"elseif not self._is_le then") + out.inc + out.puts("self:_read_be()") + out.dec + out.puts("else") + out.inc + out.puts("error(\"unable to decide endianness\")") + out.dec + out.puts("end") + } + override def readHeader(endian: Option[FixedEndian], isEmpty: Boolean): Unit = { + val suffix = endian match { + case Some(e) => s"_${e.toSuffix}" + case None => "" + } + + out.puts(s"function ${types2class(typeProvider.nowClass.name)}:_read$suffix()") + out.inc + } + override def readFooter(): Unit = { + out.dec + out.puts("end") + out.puts + } + + override def attributeDeclaration(attrName: Identifier, attrType: DataType, isNullable: Boolean): Unit = + {} + override def attributeReader(attrName: Identifier, attrType: DataType, isNullable: Boolean): Unit = + {} + + override def attrParseHybrid(leProc: () => Unit, beProc: () => Unit): Unit = { + out.puts("if self._is_le then") + out.inc + leProc() + out.dec + out.puts("else") + out.inc + beProc() + out.dec + out.puts("end") + } + + override def attrFixedContentsParse(attrName: Identifier, contents: String): Unit = + out.puts(s"${privateMemberName(attrName)} = self._io:ensure_fixed_contents($contents)") + + override def condIfHeader(expr: Ast.expr): Unit = { + out.puts(s"if ${expression(expr)} then") + out.inc + } + override def condIfFooter(expr: Ast.expr): Unit = { + out.dec + out.puts("end") + } + + override def condRepeatEosHeader(id: Identifier, io: String, dataType: DataType, needRaw: Boolean): Unit = { + if (needRaw) + out.puts(s"${privateMemberName(RawIdentifier(id))} = {}") + out.puts(s"${privateMemberName(id)} = {}") + out.puts("local i = 1") + out.puts(s"while not $io:is_eof() do") + out.inc + } + override def condRepeatEosFooter: Unit = { + out.puts("i = i + 1") + out.dec + out.puts("end") + } + + override def condRepeatExprHeader(id: Identifier, io: String, dataType: DataType, needRaw: Boolean, repeatExpr: Ast.expr): Unit = { + if (needRaw) + out.puts(s"${privateMemberName(RawIdentifier(id))} = {}") + out.puts(s"${privateMemberName(id)} = {}") + out.puts(s"for i = 1, ${expression(repeatExpr)} do") + out.inc + } + override def condRepeatExprFooter: Unit = { + out.dec + out.puts("end") + } + + override def condRepeatUntilHeader(id: Identifier, io: String, datatype: DataType, needRaw: Boolean, repeatExpr: Ast.expr): Unit = { + if (needRaw) + out.puts(s"${privateMemberName(RawIdentifier(id))} = {}") + out.puts(s"${privateMemberName(id)} = {}") + out.puts("local i = 1") + out.puts("while true do") + out.inc + } + override def condRepeatUntilFooter(id: Identifier, io: String, dataType: DataType, needRaw: Boolean, untilExpr: Ast.expr): Unit = { + typeProvider._currentIteratorType = Some(dataType) + out.puts(s"if ${expression(untilExpr)} then") + out.inc + out.puts("break") + out.dec + out.puts("end") + out.puts("i = i + 1") + out.dec + out.puts("end") + out.dec + } + + override def attrProcess(proc: ProcessExpr, varSrc: Identifier, varDest: Identifier): Unit = { + val srcName = privateMemberName(varSrc) + val destName = privateMemberName(varDest) + + proc match { + case ProcessXor(xorValue) => + val procName = translator.detectType(xorValue) match { + case _: IntType => "process_xor_one" + case _: BytesType => "process_xor_many" + } + out.puts(s"$destName = $kstreamName.$procName($srcName, ${expression(xorValue)})") + case ProcessZlib => + throw new RuntimeException("Lua zlib not supported") + case ProcessRotate(isLeft, rotValue) => + val expr = if (isLeft) { + expression(rotValue) + } else { + s"8 - (${expression(rotValue)})" + } + out.puts(s"$destName = $kstreamName.process_rotate_left($srcName, $expr, 1)") + case ProcessCustom(name, args) => + val procName = s"_process_${idToStr(varSrc)}" + + importList.add("require(\"" + s"${name.last}" + "\")") + + out.puts(s"local $procName = ${types2class(name)}(${args.map(expression).mkString(", ")})") + out.puts(s"$destName = $procName:decode($srcName)") + } + } + + override def useIO(ioEx: Ast.expr): String = { + out.puts(s"local _io = ${expression(ioEx)}") + "_io" + } + override def pushPos(io:String): Unit = + out.puts(s"local _pos = $io:pos()") + override def seek(io: String, pos: Ast.expr): Unit = + out.puts(s"$io:seek(${expression(pos)})") + override def popPos(io: String): Unit = + out.puts(s"$io:seek(_pos)") + override def alignToByte(io: String): Unit = + out.puts(s"$io:align_to_byte()") + + override def instanceHeader(className: List[String], instName: InstanceIdentifier, dataType: DataType, isNullable: Boolean): Unit = { + out.puts(s"${types2class(className)}.property.${publicMemberName(instName)} = {}") + out.puts(s"function ${types2class(className)}.property.${publicMemberName(instName)}:get()") + out.inc + } + override def instanceFooter: Unit = { + out.dec + out.puts("end") + out.puts + } + override def instanceCheckCacheAndReturn(instName: InstanceIdentifier, dataType: DataType): Unit = { + out.puts(s"if self.${idToStr(instName)} ~= nil then") + out.inc + instanceReturn(instName, dataType) + out.dec + out.puts("end") + out.puts + } + override def instanceReturn(instName: InstanceIdentifier, attrType: DataType): Unit = + out.puts(s"return ${privateMemberName(instName)}") + + override def enumDeclaration(curClass: List[String], enumName: String, enumColl: Seq[(Long, EnumValueSpec)]): Unit = { + importList.add("local enum = require(\"enum\")") + + out.puts(s"${types2class(curClass)}.${type2class(enumName)} = enum.Enum {") + out.inc + enumColl.foreach { case (id, label) => out.puts(s"${label.name} = $id,") } + out.dec + out.puts("}") + out.puts + } + + override def idToStr(id: Identifier): String = id match { + case SpecialIdentifier(name) => name + case NamedIdentifier(name) => name + case NumberedIdentifier(idx) => s"_${NumberedIdentifier.TEMPLATE}$idx" + case InstanceIdentifier(name) => s"_m_$name" + case RawIdentifier(innerId) => s"_raw_${idToStr(innerId)}" + } + override def privateMemberName(id: Identifier): String = + s"self.${idToStr(id)}" + override def publicMemberName(id: Identifier): String = id match { + case SpecialIdentifier(name) => name + case NamedIdentifier(name) => name + case InstanceIdentifier(name) => name + case RawIdentifier(innerId) => s"_raw_${publicMemberName(innerId)}" + } + override def localTemporaryName(id: Identifier): String = + s"_t_${idToStr(id)}" + + override def handleAssignmentRepeatEos(id: Identifier, expr: String): Unit = + out.puts(s"${privateMemberName(id)}[i] = $expr") + override def handleAssignmentRepeatExpr(id: Identifier, expr: String): Unit = + out.puts(s"${privateMemberName(id)}[i] = $expr") + override def handleAssignmentRepeatUntil(id: Identifier, expr: String, isRaw: Boolean): Unit = { + val tmpName = translator.doName(if (isRaw) Identifier.ITERATOR2 else Identifier.ITERATOR) + out.puts(s"$tmpName = $expr") + out.puts(s"${privateMemberName(id)}[i] = $tmpName") + } + override def handleAssignmentSimple(id: Identifier, expr: String): Unit = + out.puts(s"${privateMemberName(id)} = $expr") + + override def parseExpr(dataType: DataType, assignType: DataType, io: String, defEndian: Option[FixedEndian]): String = dataType match { + case t: ReadableType => + s"$io:read_${t.apiCall(defEndian)}()" + case blt: BytesLimitType => + s"$io:read_bytes(${expression(blt.size)})" + case _: BytesEosType => + s"$io:read_bytes_full()" + case BytesTerminatedType(terminator, include, consume, eosError, _) => + s"$io:read_bytes_term($terminator, $include, $consume, $eosError)" + case BitsType1 => + s"$io:read_bits_int(1)" + case BitsType(width: Int) => + s"$io:read_bits_int($width)" + case t: UserType => + val addParams = Utils.join(t.args.map((a) => translator.translate(a)), "", ", ", ", ") + val addArgs = if (t.isOpaque) { + "" + } else { + val parent = t.forcedParent match { + case Some(fp) => translator.translate(fp) + case None => "self" + } + val addEndian = t.classSpec.get.meta.endian match { + case Some(InheritedEndian) => ", self._is_le" + case _ => "" + } + s", $parent, self._root$addEndian" + } + s"${types2class(t.classSpec.get.name)}($addParams$io$addArgs)" + } + override def bytesPadTermExpr(expr0: String, padRight: Option[Int], terminator: Option[Int], include: Boolean): String = { + val expr1 = padRight match { + case Some(padByte) => s"$kstreamName.bytes_strip_right($expr0, $padByte)" + case None => expr0 + } + val expr2 = terminator match { + case Some(term) => s"$kstreamName.bytes_terminate($expr1, $term, $include)" + case None => expr1 + } + expr2 + } + + override def userTypeDebugRead(id: String): Unit = + out.puts(s"$id:_read()") + + override def switchStart(id: Identifier, on: Ast.expr): Unit = + out.puts(s"local _on = ${expression(on)}") + override def switchCaseFirstStart(condition: Ast.expr): Unit = { + out.puts(s"if _on == ${expression(condition)} then") + out.inc + } + override def switchCaseStart(condition: Ast.expr): Unit = { + out.puts(s"elseif _on == ${expression(condition)} then") + out.inc + } + override def switchCaseEnd(): Unit = + out.dec + override def switchElseStart(): Unit = { + out.puts("else") + out.inc + } + override def switchEnd(): Unit = + out.puts("end") + + override def allocateIO(varName: Identifier, rep: RepeatSpec): String = { + val varStr = privateMemberName(varName) + + val args = rep match { + case RepeatEos | RepeatUntil(_) => s"$varStr[#$varStr]" + case RepeatExpr(_) => s"$varStr[i]" + case NoRepeat => varStr + } + + importList.add("local stringstream = require(\"string_stream\")") + out.puts(s"local _io = $kstreamName(stringstream($args))") + "_io" + } + + override def ksErrorName(err: KSError): String = LuaCompiler.ksErrorName(err) +} + +object LuaCompiler extends LanguageCompilerStatic + with UpperCamelCaseClasses + with StreamStructNames + with ExceptionNames { + override def getCompiler( + tp: ClassTypeProvider, + config: RuntimeConfig + ): LanguageCompiler = new LuaCompiler(tp, config) + + override def kstructName: String = "KaitaiStruct" + override def kstreamName: String = "KaitaiStream" + override def ksErrorName(err: KSError): String = ??? + + def types2class(name: List[String]): String = + name.map(x => type2class(x)).mkString(".") +} diff --git a/shared/src/main/scala/io/kaitai/struct/languages/PHPCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/PHPCompiler.scala index 60bd54239..6975135dc 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/PHPCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/PHPCompiler.scala @@ -1,12 +1,12 @@ package io.kaitai.struct.languages -import io.kaitai.struct.datatype.DataType import io.kaitai.struct.datatype.DataType._ +import io.kaitai.struct.datatype.{CalcEndian, DataType, FixedEndian, InheritedEndian, KSError} import io.kaitai.struct.exprlang.Ast import io.kaitai.struct.format.{NoRepeat, RepeatEos, RepeatExpr, RepeatSpec, _} import io.kaitai.struct.languages.components._ -import io.kaitai.struct.translators.{PHPTranslator, TypeProvider} -import io.kaitai.struct.{ClassTypeProvider, LanguageOutputWriter, RuntimeConfig, Utils} +import io.kaitai.struct.translators.PHPTranslator +import io.kaitai.struct.{ClassTypeProvider, RuntimeConfig, Utils} class PHPCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) extends LanguageCompiler(typeProvider, config) @@ -67,25 +67,82 @@ class PHPCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def classFooter(name: List[String]): Unit = universalFooter - override def classConstructorHeader(name: List[String], parentClassName: List[String], rootClassName: List[String]): Unit = { - out.puts + override def classConstructorHeader(name: List[String], parentType: DataType, rootClassName: List[String], isHybrid: Boolean, params: List[ParamDefSpec]): Unit = { + typeProvider.nowClass.meta.endian match { + case Some(_: CalcEndian) | Some(InheritedEndian) => + out.puts("protected $_m__is_le;") + out.puts + case _ => + // no _is_le variable + } + + val endianAdd = if (isHybrid) ", $is_le = null" else "" + + val paramsArg = Utils.join(params.map((p) => + s"${kaitaiType2NativeType(p.dataType)} ${paramName(p.id)}" + ), "", ", ", ", ") + + // Parameter names + val pIo = paramName(IoIdentifier) + val pParent = paramName(ParentIdentifier) + val pRoot = paramName(RootIdentifier) + + // Types + val tIo = kstreamName + val tParent = kaitaiType2NativeType(parentType) + val tRoot = translator.types2classAbs(rootClassName) + out.puts( - "public function __construct(" + - kstreamName + " $io, " + - translator.types2classAbs(parentClassName) + " $parent = null, " + - translator.types2classAbs(rootClassName) + " $root = null) {" + s"public function __construct($paramsArg" + + s"$tIo $pIo, " + + s"$tParent $pParent = null, " + + s"$tRoot $pRoot = null" + endianAdd + ") {" ) out.inc - out.puts("parent::__construct($io, $parent, $root);") - out.puts("$this->_parse();") + out.puts(s"parent::__construct($pIo, $pParent, $pRoot);") + + if (isHybrid) + handleAssignmentSimple(EndianIdentifier, "$is_le") + + // Store parameters passed to us + params.foreach((p) => handleAssignmentSimple(p.id, paramName(p.id))) + } + + override def runRead(): Unit = + out.puts("$this->_read();") + + override def runReadCalc(): Unit = { + out.puts + out.puts("if (is_null($this->_m__is_le)) {") + out.inc + out.puts("throw new \\RuntimeException(\"Unable to decide on endianness\");") + out.dec + out.puts("} else if ($this->_m__is_le) {") + out.inc + out.puts("$this->_readLE();") + out.dec + out.puts("} else {") + out.inc + out.puts("$this->_readBE();") out.dec out.puts("}") + } + + override def readHeader(endian: Option[FixedEndian], isEmpty: Boolean) = { + val suffix = endian match { + case Some(e) => s"${e.toSuffix.toUpperCase}" + case None => "" + } + val access = if (config.autoRead) "private" else "public" - out.puts("private function _parse() {") + out.puts + out.puts(s"$access function _read$suffix() {") out.inc } - override def attributeDeclaration(attrName: Identifier, attrType: DataType, condSpec: ConditionalSpec): Unit = { + override def readFooter(): Unit = universalFooter + + override def attributeDeclaration(attrName: Identifier, attrType: DataType, isNullable: Boolean): Unit = { attrName match { case ParentIdentifier | RootIdentifier | IoIdentifier => // just ignore it for now @@ -94,7 +151,7 @@ class PHPCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } } - override def attributeReader(attrName: Identifier, attrType: DataType, condSpec: ConditionalSpec): Unit = { + override def attributeReader(attrName: Identifier, attrType: DataType, isNullable: Boolean): Unit = { attrName match { case ParentIdentifier | RootIdentifier => // just ignore it for now @@ -104,10 +161,24 @@ class PHPCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } override def universalDoc(doc: DocSpec): Unit = { - out.puts - out.puts( "/**") - doc.summary.foreach((summary) => out.putsLines(" * ", summary)) - out.puts( " */") + if (doc.summary.isDefined) { + out.puts + out.puts("/**") + doc.summary.foreach((summary) => out.putsLines(" * ", summary)) + out.puts(" */") + } + } + + override def attrParseHybrid(leProc: () => Unit, beProc: () => Unit): Unit = { + out.puts("if ($this->_m__is_le) {") + out.inc + leProc() + out.dec + out.puts("} else {") + out.inc + beProc() + out.dec + out.puts("}") } override def attrFixedContentsParse(attrName: Identifier, contents: String): Unit = @@ -133,11 +204,19 @@ class PHPCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) s"8 - (${expression(rotValue)})" } out.puts(s"$destName = $kstreamName::processRotateLeft($srcName, $expr, 1);") + case ProcessCustom(name, args) => + val isAbsolute = name.length > 1 + val procClass = name.map((x) => type2class(x)).mkString( + if (isAbsolute) "\\" else "", "\\", "" + ) + out.puts(s"$$_process = new $procClass(${args.map(expression).mkString(", ")});") + out.puts(s"$destName = $$_process->decode($srcName);") } } override def allocateIO(id: Identifier, rep: RepeatSpec): String = { val memberName = privateMemberName(id) + val ioName = s"$$_io_${idToStr(id)}" val args = rep match { case RepeatEos | RepeatExpr(_) => s"end($memberName)" @@ -145,8 +224,8 @@ class PHPCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) case NoRepeat => memberName } - out.puts(s"$$io = new $kstreamName($args);") - "$io" + out.puts(s"$ioName = new $kstreamName($args);") + ioName } override def useIO(ioEx: Ast.expr): String = { @@ -175,6 +254,7 @@ class PHPCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) if (needRaw) out.puts(s"${privateMemberName(RawIdentifier(id))} = [];") out.puts(s"${privateMemberName(id)} = [];") + out.puts("$i = 0;") out.puts(s"while (!$io->isEof()) {") out.inc } @@ -183,6 +263,11 @@ class PHPCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts(s"${privateMemberName(id)}[] = $expr;") } + override def condRepeatEosFooter: Unit = { + out.puts("$i++;") + super.condRepeatEosFooter + } + override def condRepeatExprHeader(id: Identifier, io: String, dataType: DataType, needRaw: Boolean, repeatExpr: Ast.expr): Unit = { if (needRaw) out.puts(s"${privateMemberName(RawIdentifier(id))} = [];") @@ -200,6 +285,7 @@ class PHPCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) if (needRaw) out.puts(s"${privateMemberName(RawIdentifier(id))} = [];") out.puts(s"${privateMemberName(id)} = [];") + out.puts("$i = 0;") out.puts("do {") out.inc } @@ -212,6 +298,7 @@ class PHPCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def condRepeatUntilFooter(id: Identifier, io: String, dataType: DataType, needRaw: Boolean, untilExpr: Ast.expr): Unit = { typeProvider._currentIteratorType = Some(dataType) + out.puts("$i++;") out.dec out.puts(s"} while (!(${expression(untilExpr)}));") } @@ -220,10 +307,13 @@ class PHPCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts(s"${privateMemberName(id)} = $expr;") } - override def parseExpr(dataType: DataType, io: String): String = { + override def handleAssignmentTempVar(dataType: DataType, id: String, expr: String): Unit = + out.puts(s"$id = $expr;") + + override def parseExpr(dataType: DataType, assignType: DataType, io: String, defEndian: Option[FixedEndian]): String = { dataType match { case t: ReadableType => - s"$io->read${Utils.capitalize(t.apiCall)}()" + s"$io->read${Utils.capitalize(t.apiCall(defEndian))}()" case blt: BytesLimitType => s"$io->readBytes(${expression(blt.size)})" case _: BytesEosType => @@ -235,6 +325,7 @@ class PHPCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) case BitsType(width: Int) => s"$io->readBitsInt($width)" case t: UserType => + val addParams = Utils.join(t.args.map((a) => translator.translate(a)), "", ", ", ", ") val addArgs = if (t.isOpaque) { "" } else { @@ -243,9 +334,13 @@ class PHPCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) case Some(fp) => translator.translate(fp) case None => "$this" } - s", $parent, ${privateMemberName(RootIdentifier)}" + val addEndian = t.classSpec.get.meta.endian match { + case Some(InheritedEndian) => s", ${privateMemberName(EndianIdentifier)}" + case _ => "" + } + s", $parent, ${privateMemberName(RootIdentifier)}$addEndian" } - s"new ${translator.types2classAbs(t.classSpec.get.name)}($io$addArgs)" + s"new ${translator.types2classAbs(t.classSpec.get.name)}($addParams$io$addArgs)" } } @@ -261,6 +356,9 @@ class PHPCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) expr2 } + override def userTypeDebugRead(id: String): Unit = + out.puts(s"$id->_read();") + override def switchStart(id: Identifier, on: Ast.expr): Unit = { val onType = translator.detectType(on) @@ -285,26 +383,27 @@ class PHPCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def switchEnd(): Unit = universalFooter - override def instanceHeader(className: List[String], instName: InstanceIdentifier, dataType: DataType): Unit = { + override def instanceHeader(className: List[String], instName: InstanceIdentifier, dataType: DataType, isNullable: Boolean): Unit = { out.puts(s"public function ${idToStr(instName)}() {") out.inc } - override def instanceCheckCacheAndReturn(instName: InstanceIdentifier): Unit = { + override def instanceCheckCacheAndReturn(instName: InstanceIdentifier, dataType: DataType): Unit = { out.puts(s"if (${privateMemberName(instName)} !== null)") out.inc - instanceReturn(instName) + instanceReturn(instName, dataType) out.dec } - override def instanceReturn(instName: InstanceIdentifier): Unit = { + override def instanceReturn(instName: InstanceIdentifier, attrType: DataType): Unit = { out.puts(s"return ${privateMemberName(instName)};") } - override def enumDeclaration(curClass: List[String], enumName: String, enumColl: Seq[(Long, String)]): Unit = { + override def enumDeclaration(curClass: List[String], enumName: String, enumColl: Seq[(Long, EnumValueSpec)]): Unit = { classHeader(curClass ::: List(enumName), None) enumColl.foreach { case (id, label) => - out.puts(s"const ${value2Const(label)} = $id;") + universalDoc(label.doc) + out.puts(s"const ${value2Const(label.name)} = $id;") } universalFooter } @@ -332,6 +431,10 @@ class PHPCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def publicMemberName(id: Identifier) = idToStr(id) + override def localTemporaryName(id: Identifier): String = s"$$_t_${idToStr(id)}" + + override def paramName(id: Identifier): String = s"$$${idToStr(id)}" + /** * Determine PHP data type corresponding to a KS data type. Currently unused due to * problems with nullable types (which were introduced only in PHP 7.1). @@ -348,17 +451,26 @@ class PHPCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) case _: StrType | _: BytesType => "string" - case t: UserType => translator.types2classAbs(t.classSpec.get.name) + case t: UserType => translator.types2classAbs(t.classSpec match { + case Some(cs) => cs.name + case None => t.name + }) case t: EnumType => "int" case ArrayType(_) => "array" + + case KaitaiStructType | CalcKaitaiStructType => kstructName + case KaitaiStreamType => kstreamName } } + + override def ksErrorName(err: KSError): String = PHPCompiler.ksErrorName(err) } object PHPCompiler extends LanguageCompilerStatic with StreamStructNames - with UpperCamelCaseClasses { + with UpperCamelCaseClasses + with ExceptionNames { override def getCompiler( tp: ClassTypeProvider, config: RuntimeConfig @@ -368,5 +480,7 @@ object PHPCompiler extends LanguageCompilerStatic override def kstructName: String = "\\Kaitai\\Struct\\Struct" + override def ksErrorName(err: KSError): String = ??? + def types2classRel(names: List[String]) = names.map(type2class).mkString("\\") } diff --git a/shared/src/main/scala/io/kaitai/struct/languages/PerlCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/PerlCompiler.scala index 969bda9f5..d2735bc2b 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/PerlCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/PerlCompiler.scala @@ -1,11 +1,11 @@ package io.kaitai.struct.languages -import io.kaitai.struct.datatype.DataType +import io.kaitai.struct.datatype.{DataType, FixedEndian, InheritedEndian, KSError} import io.kaitai.struct.datatype.DataType._ import io.kaitai.struct.exprlang.Ast import io.kaitai.struct.exprlang.Ast.expr import io.kaitai.struct.format._ -import io.kaitai.struct.languages.components._ +import io.kaitai.struct.languages.components.{ExceptionNames, _} import io.kaitai.struct.translators.{PerlTranslator, TypeProvider} import io.kaitai.struct.{ClassTypeProvider, RuntimeConfig} @@ -20,7 +20,7 @@ class PerlCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) import PerlCompiler._ - override val translator = new PerlTranslator(typeProvider) + override val translator = new PerlTranslator(typeProvider, importList) override def innerClasses = false @@ -32,15 +32,16 @@ class PerlCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def indent: String = " " override def outFileName(topClassName: String): String = s"${type2class(topClassName)}.pm" + override def outImports(topClass: ClassSpec) = + importList.toList.map((x) => s"use $x;").mkString("", "\n", "\n") + override def fileHeader(topClassName: String): Unit = { - out.puts(s"# $headerComment") - out.puts - out.puts("use strict;") - out.puts("use warnings;") - out.puts(s"use $packageName ${KSVersion.minimalRuntime.toPerlVersion};") - out.puts("use Compress::Zlib;") - out.puts("use Encode;") - out.puts("use List::Util;") + outHeader.puts(s"# $headerComment") + outHeader.puts + + importList.add("strict") + importList.add("warnings") + importList.add(s"$packageName ${KSVersion.minimalRuntime.toPerlVersion}") } override def fileFooter(topClassName: String): Unit = { @@ -71,16 +72,22 @@ class PerlCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def classFooter(name: List[String]): Unit = {} - override def classConstructorHeader(name: List[String], parentClassName: List[String], rootClassName: List[String]): Unit = { + override def classConstructorHeader(name: List[String], parentType: DataType, rootClassName: List[String], isHybrid: Boolean, params: List[ParamDefSpec]): Unit = { + val endianSuffix = if (isHybrid) ", $_is_le" else "" + out.puts out.puts("sub new {") out.inc - out.puts("my ($class, $_io, $_parent, $_root) = @_;") + out.puts("my ($class, $_io, $_parent, $_root" + endianSuffix + ") = @_;") out.puts(s"my $$self = $kstructName->new($$_io);") out.puts out.puts("bless $self, $class;") - out.puts(s"${privateMemberName(ParentIdentifier)} = $$_parent;") - out.puts(s"${privateMemberName(RootIdentifier)} = $$_root || $$self;") + handleAssignmentSimple(ParentIdentifier, "$_parent") + handleAssignmentSimple(RootIdentifier, "$_root || $self;") + + if (isHybrid) + handleAssignmentSimple(EndianIdentifier, "$_is_le") + out.puts } @@ -90,9 +97,44 @@ class PerlCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) universalFooter } - override def attributeDeclaration(attrName: Identifier, attrType: DataType, condSpec: ConditionalSpec): Unit = {} + override def runRead(): Unit = + out.puts("$self->_read();") + + override def runReadCalc(): Unit = { + val isLe = privateMemberName(EndianIdentifier) + + out.puts(s"if (!(defined $isLe)) {") + out.inc + out.puts("die \"Unable to decide on endianness\";") + out.dec + out.puts(s"} elsif ($isLe) {") + out.inc + out.puts("$self->_read_le();") + out.dec + out.puts("} else {") + out.inc + out.puts("$self->_read_be();") + out.dec + out.puts("}") + } - override def attributeReader(attrName: Identifier, attrType: DataType, condSpec: ConditionalSpec): Unit = { + override def readHeader(endian: Option[FixedEndian], isEmpty: Boolean): Unit = { + val suffix = endian match { + case Some(e) => s"_${e.toSuffix}" + case None => "" + } + out.puts + out.puts(s"sub _read$suffix {") + out.inc + out.puts("my ($self) = @_;") + out.puts + } + + override def readFooter(): Unit = universalFooter + + override def attributeDeclaration(attrName: Identifier, attrType: DataType, isNullable: Boolean): Unit = {} + + override def attributeReader(attrName: Identifier, attrType: DataType, isNullable: Boolean): Unit = { attrName match { case RootIdentifier | ParentIdentifier => // ignore, they are already defined in KaitaiStruct class @@ -108,6 +150,18 @@ class PerlCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } } + override def attrParseHybrid(leProc: () => Unit, beProc: () => Unit): Unit = { + out.puts(s"if (${privateMemberName(EndianIdentifier)}) {") + out.inc + leProc() + out.dec + out.puts("} else {") + out.inc + beProc() + out.dec + out.puts("}") + } + override def attrFixedContentsParse(attrName: Identifier, contents: String): Unit = { out.puts(s"${privateMemberName(attrName)} = $normalIO->ensure_fixed_contents($contents);") } @@ -124,6 +178,7 @@ class PerlCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } s"$destName = $kstreamName::$procName($srcName, ${expression(xorValue)});" case ProcessZlib => + importList.add("Compress::Zlib") s"$destName = Compress::Zlib::uncompress($srcName);" case ProcessRotate(isLeft, rotValue) => val expr = if (isLeft) { @@ -224,10 +279,10 @@ class PerlCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def handleAssignmentSimple(id: Identifier, expr: String): Unit = out.puts(s"${privateMemberName(id)} = $expr;") - override def parseExpr(dataType: DataType, io: String): String = { + override def parseExpr(dataType: DataType, assignType: DataType, io: String, defEndian: Option[FixedEndian]): String = { dataType match { case t: ReadableType => - s"$io->read_${t.apiCall}()" + s"$io->read_${t.apiCall(defEndian)}()" case blt: BytesLimitType => s"$io->read_bytes(${expression(blt.size)})" case _: BytesEosType => @@ -246,7 +301,11 @@ class PerlCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) case Some(fp) => translator.translate(fp) case None => "$self" } - s", $parent, ${privateMemberName(RootIdentifier)}" + val addEndian = t.classSpec.get.meta.endian match { + case Some(InheritedEndian) => s", ${privateMemberName(EndianIdentifier)}" + case _ => "" + } + s", $parent, ${privateMemberName(RootIdentifier)}$addEndian" } s"${types2class(t.classSpec.get.name)}->new($io$addArgs)" } @@ -297,26 +356,26 @@ class PerlCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) def onComparisonExpr(condition: Ast.expr) = Ast.expr.Compare(Ast.expr.Name(Ast.identifier("_on")), Ast.cmpop.Eq, condition) - override def instanceHeader(className: List[String], instName: InstanceIdentifier, dataType: DataType): Unit = { + override def instanceHeader(className: List[String], instName: InstanceIdentifier, dataType: DataType, isNullable: Boolean): Unit = { out.puts out.puts(s"sub ${instName.name} {") out.inc out.puts("my ($self) = @_;") } - override def instanceCheckCacheAndReturn(instName: InstanceIdentifier): Unit = { + override def instanceCheckCacheAndReturn(instName: InstanceIdentifier, dataType: DataType): Unit = { out.puts(s"return ${privateMemberName(instName)} if (${privateMemberName(instName)});") } - override def instanceReturn(instName: InstanceIdentifier): Unit = { + override def instanceReturn(instName: InstanceIdentifier, attrType: DataType): Unit = { out.puts(s"return ${privateMemberName(instName)};") } - override def enumDeclaration(curClass: List[String], enumName: String, enumColl: Seq[(Long, String)]): Unit = { + override def enumDeclaration(curClass: List[String], enumName: String, enumColl: Seq[(Long, EnumValueSpec)]): Unit = { out.puts enumColl.foreach { case (id, label) => - out.puts(s"our ${enumValue(enumName, label)} = $id;") + out.puts(s"our ${enumValue(enumName, label.name)} = $id;") } } @@ -336,14 +395,17 @@ class PerlCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def publicMemberName(id: Identifier): String = idToStr(id) + override def localTemporaryName(id: Identifier): String = s"$$_t_${idToStr(id)}" + def boolLiteral(b: Boolean): String = translator.doBoolLiteral(b) - def types2class(t: List[String]) = t.map(type2class).mkString("::") + override def ksErrorName(err: KSError): String = PerlCompiler.ksErrorName(err) } object PerlCompiler extends LanguageCompilerStatic with UpperCamelCaseClasses - with StreamStructNames { + with StreamStructNames + with ExceptionNames { override def getCompiler( tp: ClassTypeProvider, config: RuntimeConfig @@ -352,4 +414,7 @@ object PerlCompiler extends LanguageCompilerStatic def packageName: String = "IO::KaitaiStruct" override def kstreamName: String = s"$packageName::Stream" override def kstructName: String = s"$packageName::Struct" + override def ksErrorName(err: KSError): String = ??? + + def types2class(t: List[String]): String = t.map(type2class).mkString("::") } diff --git a/shared/src/main/scala/io/kaitai/struct/languages/PythonCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/PythonCompiler.scala index 18286040e..2ccd6f1c3 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/PythonCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/PythonCompiler.scala @@ -1,13 +1,13 @@ package io.kaitai.struct.languages +import io.kaitai.struct.datatype.DataType._ +import io.kaitai.struct.datatype.{DataType, FixedEndian, InheritedEndian, KSError, UndecidedEndiannessError} import io.kaitai.struct.exprlang.Ast import io.kaitai.struct.exprlang.Ast.expr -import io.kaitai.struct.datatype.DataType -import io.kaitai.struct.datatype.DataType._ import io.kaitai.struct.format._ import io.kaitai.struct.languages.components._ -import io.kaitai.struct.translators.{PythonTranslator, TypeProvider} -import io.kaitai.struct.{ClassTypeProvider, LanguageOutputWriter, RuntimeConfig} +import io.kaitai.struct.translators.PythonTranslator +import io.kaitai.struct.{ClassTypeProvider, RuntimeConfig, StringLanguageOutputWriter, Utils} class PythonCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) extends LanguageCompiler(typeProvider, config) @@ -18,11 +18,14 @@ class PythonCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) with EveryReadIsExpression with AllocateIOLocalVar with FixedContentsUsingArrayByteLiteral + with UniversalDoc with NoNeedForFullClassPath { import PythonCompiler._ - override val translator = new PythonTranslator(typeProvider) + override val translator = new PythonTranslator(typeProvider, importList) + + override def innerDocstrings = true override def universalFooter: Unit = { out.dec @@ -32,22 +35,23 @@ class PythonCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def indent: String = " " override def outFileName(topClassName: String): String = s"$topClassName.py" + override def outImports(topClass: ClassSpec) = + importList.toList.mkString("", "\n", "\n") + override def fileHeader(topClassName: String): Unit = { - out.puts(s"# $headerComment") - out.puts - out.puts("import array") - out.puts("import struct") - out.puts("import zlib") - out.puts("from enum import Enum") - out.puts("from pkg_resources import parse_version") - out.puts - out.puts(s"from kaitaistruct import __version__ as ks_version, $kstructName, $kstreamName, BytesIO") + outHeader.puts(s"# $headerComment") + outHeader.puts + + importList.add("from pkg_resources import parse_version") + importList.add("import kaitaistruct") + importList.add(s"from kaitaistruct import $kstructName, $kstreamName, BytesIO") + out.puts out.puts // API compatibility check out.puts( - "if parse_version(ks_version) < parse_version('" + + "if parse_version(kaitaistruct.__version__) < parse_version('" + KSVersion.minimalRuntime + "'):" ) @@ -55,7 +59,7 @@ class PythonCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts( "raise Exception(\"Incompatible Kaitai Struct Python API: " + KSVersion.minimalRuntime + - " or later is required, but you have %s\" % (ks_version))" + " or later is required, but you have %s\" % (kaitaistruct.__version__))" ) out.dec out.puts @@ -63,7 +67,13 @@ class PythonCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def opaqueClassDeclaration(classSpec: ClassSpec): Unit = { val name = classSpec.name.head - out.puts(s"from $name import ${type2class(name)}") + out.puts( + if (config.pythonPackage.nonEmpty) { + s"from ${config.pythonPackage} import $name" + } else { + s"import $name" + } + ) } override def classHeader(name: String): Unit = { @@ -71,38 +81,141 @@ class PythonCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.inc } - override def classConstructorHeader(name: String, parentClassName: String, rootClassName: String): Unit = { - out.puts("def __init__(self, _io, _parent=None, _root=None):") + override def classConstructorHeader(name: String, parentType: DataType, rootClassName: String, isHybrid: Boolean, params: List[ParamDefSpec]): Unit = { + val endianAdd = if (isHybrid) ", _is_le=None" else "" + val paramsList = Utils.join(params.map((p) => paramName(p.id)), ", ", ", ", "") + + out.puts(s"def __init__(self$paramsList, _io, _parent=None, _root=None$endianAdd):") out.inc out.puts("self._io = _io") out.puts("self._parent = _parent") out.puts("self._root = _root if _root else self") + + if (isHybrid) + out.puts("self._is_le = _is_le") + + // Store parameters passed to us + params.foreach((p) => handleAssignmentSimple(p.id, paramName(p.id))) + + if (config.readStoresPos) { + importList.add("import collections") + out.puts("self._debug = collections.defaultdict(dict)") + } + } + + override def runRead(): Unit = { + out.puts("self._read()") + } + + override def runReadCalc(): Unit = { + out.puts(s"if not hasattr(self, '_is_le'):") + out.inc + out.puts(s"raise ${ksErrorName(UndecidedEndiannessError)}(" + "\"" + typeProvider.nowClass.path.mkString("/", "/", "") + "\")") + out.dec + out.puts(s"elif self._is_le == True:") + out.inc + out.puts("self._read_le()") + out.dec + out.puts("elif self._is_le == False:") + out.inc + out.puts("self._read_be()") + out.dec + } + + override def readHeader(endian: Option[FixedEndian], isEmpty: Boolean): Unit = { + val suffix = endian match { + case Some(e) => s"_${e.toSuffix}" + case None => "" + } + out.puts(s"def _read$suffix(self):") + out.inc + if (isEmpty) + out.puts("pass") } - override def attributeDeclaration(attrName: Identifier, attrType: DataType, condSpec: ConditionalSpec): Unit = {} + override def readFooter() = universalFooter + + override def attributeDeclaration(attrName: Identifier, attrType: DataType, isNullable: Boolean): Unit = {} - override def attributeReader(attrName: Identifier, attrType: DataType, condSpec: ConditionalSpec): Unit = {} + override def attributeReader(attrName: Identifier, attrType: DataType, isNullable: Boolean): Unit = {} + + override def universalDoc(doc: DocSpec): Unit = { + val docStr = doc.summary match { + case Some(summary) => + val lastChar = summary.last + if (lastChar == '.' || lastChar == '\n') { + summary + } else { + summary + "." + } + case None => + "" + } + + val extraNewline = if (docStr.isEmpty || docStr.last == '\n') "" else "\n" + val refStr = doc.ref.map { + case TextRef(text) => + val seeAlso = new StringLanguageOutputWriter("") + seeAlso.putsLines(" ", text) + s"$extraNewline\n.. seealso::\n${seeAlso.result}" + case ref: UrlRef => + val seeAlso = new StringLanguageOutputWriter("") + seeAlso.putsLines(" ", s"${ref.text} - ${ref.url}") + s"$extraNewline\n.. seealso::\n${seeAlso.result}" + }.mkString("\n") + + out.putsLines("", "\"\"\"" + docStr + refStr + "\"\"\"") + } override def attrFixedContentsParse(attrName: Identifier, contents: String): Unit = out.puts(s"${privateMemberName(attrName)} = self._io.ensure_fixed_contents($contents)") + override def attrParseHybrid(leProc: () => Unit, beProc: () => Unit): Unit = { + out.puts("if self._is_le:") + out.inc + leProc() + out.dec + out.puts("else:") + out.inc + beProc() + out.dec + } + override def attrProcess(proc: ProcessExpr, varSrc: Identifier, varDest: Identifier): Unit = { + val srcName = privateMemberName(varSrc) + val destName = privateMemberName(varDest) + proc match { case ProcessXor(xorValue) => val procName = translator.detectType(xorValue) match { case _: IntType => "process_xor_one" case _: BytesType => "process_xor_many" } - out.puts(s"${privateMemberName(varDest)} = $kstreamName.$procName(${privateMemberName(varSrc)}, ${expression(xorValue)})") + out.puts(s"$destName = $kstreamName.$procName($srcName, ${expression(xorValue)})") case ProcessZlib => - out.puts(s"${privateMemberName(varDest)} = zlib.decompress(${privateMemberName(varSrc)})") + importList.add("import zlib") + out.puts(s"$destName = zlib.decompress($srcName)") case ProcessRotate(isLeft, rotValue) => val expr = if (isLeft) { expression(rotValue) } else { s"8 - (${expression(rotValue)})" } - out.puts(s"${privateMemberName(varDest)} = $kstreamName.process_rotate_left(${privateMemberName(varSrc)}, $expr, 1)") + out.puts(s"$destName = $kstreamName.process_rotate_left($srcName, $expr, 1)") + case ProcessCustom(name, args) => + val procClass = if (name.length == 1) { + val onlyName = name.head + val className = type2class(onlyName) + importList.add(s"from $onlyName import $className") + className + } else { + val pkgName = name.init.mkString(".") + importList.add(s"import $pkgName") + s"$pkgName.${type2class(name.last)}" + } + + out.puts(s"_process = $procClass(${args.map(expression).mkString(", ")})") + out.puts(s"$destName = _process.decode($srcName)") } } @@ -110,6 +223,7 @@ class PythonCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def allocateIO(varName: Identifier, rep: RepeatSpec): String = { val varStr = privateMemberName(varName) + val ioName = s"_io_${idToStr(varName)}" val args = rep match { case RepeatEos | RepeatUntil(_) => s"$varStr[-1]" @@ -117,8 +231,8 @@ class PythonCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) case NoRepeat => varStr } - out.puts(s"io = $kstreamName(BytesIO($args))") - "io" + out.puts(s"$ioName = $kstreamName(BytesIO($args))") + ioName } override def useIO(ioEx: expr): String = { @@ -138,6 +252,40 @@ class PythonCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def alignToByte(io: String): Unit = out.puts(s"$io.align_to_byte()") + override def attrDebugStart(attrId: Identifier, attrType: DataType, ios: Option[String], rep: RepeatSpec): Unit = { + ios.foreach { (io) => + val name = attrId match { + case _: RawIdentifier | _: SpecialIdentifier => return + case _ => idToStr(attrId) + } + rep match { + case NoRepeat => + out.puts(s"self._debug['$name']['start'] = $io.pos()") + case _: RepeatExpr | RepeatEos | _: RepeatUntil => + out.puts(s"if not 'arr' in self._debug['$name']:") + out.inc + out.puts(s"self._debug['$name']['arr'] = []") + out.dec + out.puts(s"self._debug['$name']['arr'].append({'start': $io.pos()})") + } + } + } + + override def attrDebugEnd(attrId: Identifier, attrType: DataType, io: String, rep: RepeatSpec): Unit = { + val name = attrId match { + case _: RawIdentifier | _: SpecialIdentifier => return + case _ => idToStr(attrId) + } + rep match { + case NoRepeat => + out.puts(s"self._debug['$name']['end'] = $io.pos()") + case _: RepeatExpr => + out.puts(s"self._debug['$name']['arr'][i]['end'] = $io.pos()") + case RepeatEos | _: RepeatUntil => + out.puts(s"self._debug['$name']['arr'][len(${privateMemberName(attrId)}) - 1]['end'] = $io.pos()") + } + } + override def condIfHeader(expr: Ast.expr): Unit = { out.puts(s"if ${expression(expr)}:") out.inc @@ -147,11 +295,16 @@ class PythonCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) if (needRaw) out.puts(s"${privateMemberName(RawIdentifier(id))} = []") out.puts(s"${privateMemberName(id)} = []") + out.puts("i = 0") out.puts(s"while not $io.is_eof():") out.inc } override def handleAssignmentRepeatEos(id: Identifier, expr: String): Unit = out.puts(s"${privateMemberName(id)}.append($expr)") + override def condRepeatEosFooter: Unit = { + out.puts("i += 1") + universalFooter + } override def condRepeatExprHeader(id: Identifier, io: String, dataType: DataType, needRaw: Boolean, repeatExpr: expr): Unit = { if (needRaw) @@ -167,6 +320,7 @@ class PythonCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) if (needRaw) out.puts(s"${privateMemberName(RawIdentifier(id))} = []") out.puts(s"${privateMemberName(id)} = []") + out.puts("i = 0") out.puts("while True:") out.inc } @@ -183,16 +337,20 @@ class PythonCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.inc out.puts("break") out.dec + out.puts("i += 1") out.dec } override def handleAssignmentSimple(id: Identifier, expr: String): Unit = out.puts(s"${privateMemberName(id)} = $expr") - override def parseExpr(dataType: DataType, io: String): String = { + override def handleAssignmentTempVar(dataType: DataType, id: String, expr: String): Unit = + out.puts(s"$id = $expr") + + override def parseExpr(dataType: DataType, assignType: DataType, io: String, defEndian: Option[FixedEndian]): String = { dataType match { case t: ReadableType => - s"$io.read_${t.apiCall}()" + s"$io.read_${t.apiCall(defEndian)}()" case blt: BytesLimitType => s"$io.read_bytes(${expression(blt.size)})" case _: BytesEosType => @@ -204,6 +362,7 @@ class PythonCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) case BitsType(width: Int) => s"$io.read_bits_int($width)" case t: UserType => + val addParams = Utils.join(t.args.map((a) => translator.translate(a)), "", ", ", ", ") val addArgs = if (t.isOpaque) { "" } else { @@ -211,9 +370,13 @@ class PythonCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) case Some(fp) => translator.translate(fp) case None => "self" } - s", $parent, self._root" + val addEndian = t.classSpec.get.meta.endian match { + case Some(InheritedEndian) => ", self._is_le" + case _ => "" + } + s", $parent, self._root$addEndian" } - s"${types2class(t.classSpec.get.name)}($io$addArgs)" + s"${userType2class(t)}($addParams$io$addArgs)" } } @@ -229,6 +392,9 @@ class PythonCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) expr2 } + override def userTypeDebugRead(id: String): Unit = + out.puts(s"$id._read()") + override def switchStart(id: Identifier, on: Ast.expr): Unit = { out.puts(s"_on = ${expression(on)}") } @@ -253,27 +419,29 @@ class PythonCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def switchEnd(): Unit = {} - override def instanceHeader(className: String, instName: InstanceIdentifier, dataType: DataType): Unit = { + override def instanceHeader(className: String, instName: InstanceIdentifier, dataType: DataType, isNullable: Boolean): Unit = { out.puts("@property") out.puts(s"def ${publicMemberName(instName)}(self):") out.inc } - override def instanceCheckCacheAndReturn(instName: InstanceIdentifier): Unit = { + override def instanceCheckCacheAndReturn(instName: InstanceIdentifier, dataType: DataType): Unit = { out.puts(s"if hasattr(self, '${idToStr(instName)}'):") out.inc - instanceReturn(instName) + instanceReturn(instName, dataType) out.dec out.puts } - override def instanceReturn(instName: InstanceIdentifier): Unit = { + override def instanceReturn(instName: InstanceIdentifier, attrType: DataType): Unit = { // not very efficient, probably should be some other way to do that, but for now it will do: // workaround to avoid Python generating an "AttributeError: instance has no attribute" out.puts(s"return ${privateMemberName(instName)} if hasattr(self, '${idToStr(instName)}') else None") } override def enumDeclaration(curClass: String, enumName: String, enumColl: Seq[(Long, String)]): Unit = { + importList.add("from enum import Enum") + out.puts out.puts(s"class ${type2class(enumName)}(Enum):") out.inc @@ -281,6 +449,11 @@ class PythonCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.dec } + override def debugClassSequence(seq: List[AttrSpec]) = { + val seqStr = seq.map((attr) => "\"" + idToStr(attr.id) + "\"").mkString(", ") + out.puts(s"SEQ_FIELDS = [$seqStr]") + } + def bool2Py(b: Boolean): String = if (b) { "True" } else { "False" } def idToStr(id: Identifier): String = { @@ -303,11 +476,41 @@ class PythonCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) case RawIdentifier(innerId) => s"_raw_${publicMemberName(innerId)}" } } + + override def localTemporaryName(id: Identifier): String = s"_t_${idToStr(id)}" + + override def ksErrorName(err: KSError): String = PythonCompiler.ksErrorName(err) + + override def attrValidateExpr( + attrId: Identifier, + attrType: DataType, + checkExpr: Ast.expr, + errName: String, + errArgs: List[Ast.expr] + ): Unit = { + val errArgsStr = errArgs.map(translator.translate).mkString(", ") + out.puts(s"if not ${translator.translate(checkExpr)}:") + out.inc + out.puts(s"raise $errName($errArgsStr)") + out.dec + } + + def userType2class(t: UserType): String = { + val name = t.classSpec.get.name + val firstName = name.head + val prefix = if (t.isOpaque && firstName != translator.provider.nowClass.name.head) { + s"$firstName." + } else { + "" + } + s"$prefix${types2class(name)}" + } } object PythonCompiler extends LanguageCompilerStatic with UpperCamelCaseClasses - with StreamStructNames { + with StreamStructNames + with ExceptionNames { override def getCompiler( tp: ClassTypeProvider, config: RuntimeConfig @@ -315,6 +518,7 @@ object PythonCompiler extends LanguageCompilerStatic override def kstreamName: String = "KaitaiStream" override def kstructName: String = "KaitaiStruct" + override def ksErrorName(err: KSError): String = s"kaitaistruct.${err.name}" def types2class(name: List[String]): String = { if (name.size > 1) { diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RubyCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RubyCompiler.scala index 8aa4cbe9d..2208be50f 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RubyCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RubyCompiler.scala @@ -1,13 +1,13 @@ package io.kaitai.struct.languages +import io.kaitai.struct.datatype.DataType._ +import io.kaitai.struct.datatype._ import io.kaitai.struct.exprlang.Ast import io.kaitai.struct.exprlang.Ast.expr -import io.kaitai.struct.datatype.DataType -import io.kaitai.struct.datatype.DataType._ import io.kaitai.struct.format._ import io.kaitai.struct.languages.components._ -import io.kaitai.struct.translators.{RubyTranslator, TypeProvider} -import io.kaitai.struct.{ClassTypeProvider, LanguageOutputWriter, RuntimeConfig} +import io.kaitai.struct.translators.RubyTranslator +import io.kaitai.struct.{ClassTypeProvider, RuntimeConfig, Utils} class RubyCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) extends LanguageCompiler(typeProvider, config) @@ -33,11 +33,15 @@ class RubyCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def outFileName(topClassName: String): String = s"$topClassName.rb" override def indent: String = " " + override def outImports(topClass: ClassSpec) = + importList.toList.map((x) => s"require '$x'").mkString("\n") + "\n" + override def fileHeader(topClassName: String): Unit = { - out.puts(s"# $headerComment") - out.puts - out.puts("require 'kaitai/struct/struct'") - out.puts("require 'zlib'") // TODO: add only if actually used + outHeader.puts(s"# $headerComment") + outHeader.puts + + importList.add("kaitai/struct/struct") + out.puts // API compatibility check @@ -60,37 +64,82 @@ class RubyCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def classHeader(name: String): Unit = { out.puts(s"class ${type2class(name)} < $kstructName") out.inc - if (debug) + if (config.readStoresPos) out.puts("attr_reader :_debug") } - override def classConstructorHeader(name: String, parentClassName: String, rootClassName: String): Unit = { - out.puts("def initialize(_io, _parent = nil, _root = self)") + override def classConstructorHeader(name: String, parentType: DataType, rootClassName: String, isHybrid: Boolean, params: List[ParamDefSpec]): Unit = { + val endianSuffix = if (isHybrid) { + ", _is_le = nil" + } else { + "" + } + + val paramsList = Utils.join(params.map((p) => paramName(p.id)), ", ", ", ", "") + + out.puts(s"def initialize(_io, _parent = nil, _root = self$endianSuffix$paramsList)") out.inc out.puts("super(_io, _parent, _root)") - if (debug) { + + if (isHybrid) { + out.puts("@_is_le = _is_le") + } + + // Store parameters passed to us + params.foreach((p) => handleAssignmentSimple(p.id, paramName(p.id))) + + if (config.readStoresPos) { out.puts("@_debug = {}") - out.dec - out.puts("end") - out.puts - out.puts("def _read") - out.inc } } - override def classConstructorFooter: Unit = { - if (debug) { - // Actually, it's not constructor in debug mode, but a "_read" method. Make sure it returns an instance of the - // class, just as normal Foo.new call does. - out.puts - out.puts("self") + override def runRead(): Unit = { + out.puts("_read") + } + + override def runReadCalc(): Unit = { + out.puts + out.puts(s"if @_is_le == true") + out.inc + out.puts("_read_le") + out.dec + out.puts("elsif @_is_le == false") + out.inc + out.puts("_read_be") + out.dec + out.puts("else") + out.inc + out.puts(s"raise ${ksErrorName(UndecidedEndiannessError)}.new(" + "\"" + typeProvider.nowClass.path.mkString("/", "/", "") + "\")") + out.dec + out.puts("end") + } + + override def readHeader(endian: Option[FixedEndian], isEmpty: Boolean) = { + val suffix = endian match { + case Some(e) => s"_${e.toSuffix}" + case None => "" } + out.puts + out.puts(s"def _read$suffix") + out.inc + } + + override def readFooter() = { + // This is required for debug mode to be able to do stuff like: + // + // obj = Obj.new(...)._read + // + // i.e. drop-in replacement of non-debug mode invocation: + // + // obj = Obj.new(...) + out.puts("self") + universalFooter } - override def attributeDeclaration(attrName: Identifier, attrType: DataType, condSpec: ConditionalSpec): Unit = {} + override def attributeDeclaration(attrName: Identifier, attrType: DataType, isNullable: Boolean): Unit = {} - override def attributeReader(attrName: Identifier, attrType: DataType, condSpec: ConditionalSpec): Unit = { + override def attributeReader(attrName: Identifier, attrType: DataType, isNullable: Boolean): Unit = { attrName match { case RootIdentifier | ParentIdentifier => // ignore, they are already added in Kaitai::Struct::Struct @@ -103,18 +152,28 @@ class RubyCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts out.puts("##") - doc.summary.foreach((summary) => out.putsLines("# ", summary)) + doc.summary.foreach(summary => out.putsLines("# ", summary)) - doc.ref match { + doc.ref.foreach { case TextRef(text) => out.putsLines("# ", s"@see '' $text", " ") case UrlRef(url, text) => out.putsLines("# ", s"@see $url $text", " ") - case NoRef => - // do nothing } } + override def attrParseHybrid(leProc: () => Unit, beProc: () => Unit): Unit = { + out.puts("if @_is_le") + out.inc + leProc() + out.dec + out.puts("else") + out.inc + beProc() + out.dec + out.puts("end") + } + override def attrFixedContentsParse(attrName: Identifier, contents: String): Unit = out.puts(s"${privateMemberName(attrName)} = $normalIO.ensure_fixed_contents($contents)") @@ -130,6 +189,7 @@ class RubyCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } s"$destName = $kstreamName::$procName($srcName, ${expression(xorValue)})" case ProcessZlib => + importList.add("zlib") s"$destName = Zlib::Inflate.inflate($srcName)" case ProcessRotate(isLeft, rotValue) => val expr = if (isLeft) { @@ -138,11 +198,16 @@ class RubyCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) s"8 - (${expression(rotValue)})" } s"$destName = $kstreamName::process_rotate_left($srcName, $expr, 1)" + case ProcessCustom(name, args) => + val procClass = name.map((x) => type2class(x)).mkString("::") + out.puts(s"_process = $procClass.new(${args.map(expression).mkString(", ")})") + s"$destName = _process.decode($srcName)" }) } override def allocateIO(id: Identifier, rep: RepeatSpec): String = { val memberName = privateMemberName(id) + val ioName = s"_io_${idToStr(id)}" val args = rep match { case RepeatEos | RepeatUntil(_) => s"$memberName.last" @@ -150,8 +215,8 @@ class RubyCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) case NoRepeat => s"$memberName" } - out.puts(s"io = $kstreamName.new($args)") - "io" + out.puts(s"$ioName = $kstreamName.new($args)") + ioName } override def useIO(ioEx: expr): String = { @@ -213,11 +278,16 @@ class RubyCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts(s"${privateMemberName(RawIdentifier(id))} = []") out.puts(s"${privateMemberName(id)} = []") + out.puts("i = 0") out.puts(s"while not $io.eof?") out.inc } override def handleAssignmentRepeatEos(id: Identifier, expr: String): Unit = out.puts(s"${privateMemberName(id)} << $expr") + override def condRepeatEosFooter: Unit = { + out.puts("i += 1") + super.condRepeatEosFooter + } override def condRepeatExprHeader(id: Identifier, io: String, dataType: DataType, needRaw: Boolean, repeatExpr: expr): Unit = { if (needRaw) @@ -237,6 +307,7 @@ class RubyCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) if (needRaw) out.puts(s"${privateMemberName(RawIdentifier(id))} = []") out.puts(s"${privateMemberName(id)} = []") + out.puts("i = 0") out.puts("begin") out.inc } @@ -249,6 +320,7 @@ class RubyCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def condRepeatUntilFooter(id: Identifier, io: String, dataType: DataType, needRaw: Boolean, untilExpr: expr): Unit = { typeProvider._currentIteratorType = Some(dataType) + out.puts("i += 1") out.dec out.puts(s"end until ${expression(untilExpr)}") } @@ -259,10 +331,10 @@ class RubyCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def handleAssignmentTempVar(dataType: DataType, id: String, expr: String): Unit = out.puts(s"$id = $expr") - override def parseExpr(dataType: DataType, io: String): String = { + override def parseExpr(dataType: DataType, assignType: DataType, io: String, defEndian: Option[FixedEndian]): String = { dataType match { case t: ReadableType => - s"$io.read_${t.apiCall}" + s"$io.read_${t.apiCall(defEndian)}" case blt: BytesLimitType => s"$io.read_bytes(${expression(blt.size)})" case _: BytesEosType => @@ -274,6 +346,7 @@ class RubyCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) case BitsType(width: Int) => s"$io.read_bits_int($width)" case t: UserType => + val addParams = Utils.join(t.args.map((a) => translator.translate(a)), ", ", ", ", "") val addArgs = if (t.isOpaque) { "" } else { @@ -281,9 +354,13 @@ class RubyCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) case Some(fp) => translator.translate(fp) case None => "self" } - s", $parent, @_root" + val addEndian = t.classSpec.get.meta.endian match { + case Some(InheritedEndian) => ", @_is_le" + case _ => "" + } + s", $parent, @_root$addEndian" } - s"${type2class(t.name.last)}.new($io$addArgs)" + s"${types2class(t.name)}.new($io$addArgs$addParams)" } } @@ -321,16 +398,16 @@ class RubyCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def switchEnd(): Unit = out.puts("end") - override def instanceHeader(className: String, instName: InstanceIdentifier, dataType: DataType): Unit = { + override def instanceHeader(className: String, instName: InstanceIdentifier, dataType: DataType, isNullable: Boolean): Unit = { out.puts(s"def ${instName.name}") out.inc } - override def instanceCheckCacheAndReturn(instName: InstanceIdentifier): Unit = { + override def instanceCheckCacheAndReturn(instName: InstanceIdentifier, dataType: DataType): Unit = { out.puts(s"return ${privateMemberName(instName)} unless ${privateMemberName(instName)}.nil?") } - override def instanceReturn(instName: InstanceIdentifier): Unit = { + override def instanceReturn(instName: InstanceIdentifier, attrType: DataType): Unit = { out.puts(privateMemberName(instName)) } @@ -372,10 +449,28 @@ class RubyCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def privateMemberName(id: Identifier): String = s"@${idToStr(id)}" override def publicMemberName(id: Identifier): String = idToStr(id) + + override def localTemporaryName(id: Identifier): String = s"_t_${idToStr(id)}" + + override def ksErrorName(err: KSError): String = RubyCompiler.ksErrorName(err) + + override def attrValidateExpr( + attrId: Identifier, + attrType: DataType, + checkExpr: Ast.expr, + errName: String, + errArgs: List[Ast.expr] + ): Unit = { + val errArgsStr = errArgs.map(translator.translate).mkString(", ") + out.puts(s"raise $errName.new($errArgsStr) if not ${translator.translate(checkExpr)}") + } + + def types2class(names: List[String]) = names.map(type2class).mkString("::") } object RubyCompiler extends LanguageCompilerStatic - with StreamStructNames { + with StreamStructNames + with ExceptionNames { override def getCompiler( tp: ClassTypeProvider, config: RuntimeConfig @@ -383,6 +478,7 @@ object RubyCompiler extends LanguageCompilerStatic override def kstreamName: String = "Kaitai::Struct::Stream" override def kstructName: String = "Kaitai::Struct::Struct" + override def ksErrorName(err: KSError): String = s"Kaitai::Struct::${err.name}" def inverseEnumName(enumName: String) = s"I__$enumName" } diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala new file mode 100644 index 000000000..3ba00162e --- /dev/null +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -0,0 +1,609 @@ +package io.kaitai.struct.languages + +import io.kaitai.struct.datatype.DataType._ +import io.kaitai.struct.datatype.{DataType, FixedEndian, InheritedEndian, KSError} +import io.kaitai.struct.exprlang.Ast +import io.kaitai.struct.format.{NoRepeat, RepeatEos, RepeatExpr, RepeatSpec, _} +import io.kaitai.struct.languages.components._ +import io.kaitai.struct.translators.RustTranslator +import io.kaitai.struct.{ClassTypeProvider, RuntimeConfig, Utils} + +class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) + extends LanguageCompiler(typeProvider, config) + with ObjectOrientedLanguage + with UpperCamelCaseClasses + with SingleOutputFile + with AllocateIOLocalVar + with UniversalFooter + with UniversalDoc + with FixedContentsUsingArrayByteLiteral + with EveryReadIsExpression { + + import RustCompiler._ + + override def innerClasses = false + + override def innerEnums = false + + override val translator: RustTranslator = new RustTranslator(typeProvider, config) + + override def universalFooter: Unit = { + out.dec + out.puts("}") + } + + override def outImports(topClass: ClassSpec) = + importList.toList.map((x) => s"use $x;").mkString("", "\n", "\n") + + override def indent: String = " " + override def outFileName(topClassName: String): String = s"$topClassName.rs" + + override def fileHeader(topClassName: String): Unit = { + outHeader.puts(s"// $headerComment") + outHeader.puts + + importList.add("std::option::Option") + importList.add("std::boxed::Box") + importList.add("std::io::Result") + importList.add("std::io::Cursor") + importList.add("std::vec::Vec") + importList.add("std::default::Default") + importList.add("kaitai_struct::KaitaiStream") + importList.add("kaitai_struct::KaitaiStruct") + + out.puts + } + + override def opaqueClassDeclaration(classSpec: ClassSpec): Unit = { + val name = type2class(classSpec.name.last) + val pkg = type2classAbs(classSpec.name) + + importList.add(s"$pkg::$name") + } + + override def classHeader(name: List[String]): Unit = + classHeader(name, Some(kstructName)) + + def classHeader(name: List[String], parentClass: Option[String]): Unit = { + out.puts("#[derive(Default)]") + out.puts(s"pub struct ${type2class(name)} {") + } + + override def classFooter(name: List[String]): Unit = universalFooter + + override def classConstructorHeader(name: List[String], parentType: DataType, rootClassName: List[String], isHybrid: Boolean, params: List[ParamDefSpec]): Unit = { + out.puts("}") + out.puts + + out.puts(s"impl KaitaiStruct for ${type2class(name)} {") + out.inc + + // Parameter names + val pIo = paramName(IoIdentifier) + val pParent = paramName(ParentIdentifier) + val pRoot = paramName(RootIdentifier) + + // Types + val tIo = kstreamName + val tParent = kaitaiType2NativeType(parentType) + + out.puts(s"fn new(stream: &mut S,") + out.puts(s" _parent: &Option>,") + out.puts(s" _root: &Option>)") + out.puts(s" -> Result") + out.inc + out.puts(s"where Self: Sized {") + + out.puts(s"let mut s: Self = Default::default();") + out.puts + + out.puts(s"s.stream = stream;") + + out.puts(s"s.read(stream, _parent, _root)?;") + out.puts + + out.puts("Ok(s)") + out.dec + out.puts("}") + out.puts + } + + override def runRead(): Unit = { + + } + + override def runReadCalc(): Unit = { + + } + + override def readHeader(endian: Option[FixedEndian], isEmpty: Boolean) = { + out.puts + out.puts(s"fn read(&mut self,") + out.puts(s" stream: &mut S,") + out.puts(s" _parent: &Option>,") + out.puts(s" _root: &Option>)") + out.puts(s" -> Result<()>") + out.inc + out.puts(s"where Self: Sized {") + } + + override def readFooter(): Unit = { + out.puts + out.puts("Ok(())") + out.dec + out.puts("}") + } + + override def attributeDeclaration(attrName: Identifier, attrType: DataType, isNullable: Boolean): Unit = { + attrName match { + case ParentIdentifier | RootIdentifier | IoIdentifier => + // just ignore it for now + case IoIdentifier => + out.puts(s" stream: ${kaitaiType2NativeType(attrType)},") + case _ => + out.puts(s" pub ${idToStr(attrName)}: ${kaitaiType2NativeType(attrType)},") + } + } + + override def attributeReader(attrName: Identifier, attrType: DataType, isNullable: Boolean): Unit = { + + } + + override def universalDoc(doc: DocSpec): Unit = { + if (doc.summary.isDefined) { + out.puts + out.puts("/*") + doc.summary.foreach((summary) => out.putsLines(" * ", summary)) + out.puts(" */") + } + } + + override def attrParseHybrid(leProc: () => Unit, beProc: () => Unit): Unit = { + out.puts("if ($this->_m__is_le) {") + out.inc + leProc() + out.dec + out.puts("} else {") + out.inc + beProc() + out.dec + out.puts("}") + } + + override def attrFixedContentsParse(attrName: Identifier, contents: String): Unit = + out.puts(s"${privateMemberName(attrName)} = $normalIO.ensureFixedContents($contents);") + + override def attrProcess(proc: ProcessExpr, varSrc: Identifier, varDest: Identifier): Unit = { + val srcName = privateMemberName(varSrc) + val destName = privateMemberName(varDest) + + proc match { + case ProcessXor(xorValue) => + val procName = translator.detectType(xorValue) match { + case _: IntType => "processXorOne" + case _: BytesType => "processXorMany" + } + out.puts(s"$destName = $kstreamName::$procName($srcName, ${expression(xorValue)});") + case ProcessZlib => + out.puts(s"$destName = $kstreamName::processZlib($srcName);") + case ProcessRotate(isLeft, rotValue) => + val expr = if (isLeft) { + expression(rotValue) + } else { + s"8 - (${expression(rotValue)})" + } + out.puts(s"$destName = $kstreamName::processRotateLeft($srcName, $expr, 1);") + case ProcessCustom(name, args) => + val procClass = if (name.length == 1) { + val onlyName = name.head + val className = type2class(onlyName) + importList.add(s"$onlyName::$className") + className + } else { + val pkgName = type2classAbs(name.init) + val className = type2class(name.last) + importList.add(s"$pkgName::$className") + s"$pkgName::$className" + } + + out.puts(s"let _process = $procClass::new(${args.map(expression).mkString(", ")});") + out.puts(s"$destName = _process.decode($srcName);") + } + } + + override def allocateIO(id: Identifier, rep: RepeatSpec): String = { + val memberName = privateMemberName(id) + + val args = rep match { + case RepeatEos | RepeatExpr(_) => s"$memberName.last()" + case RepeatUntil(_) => translator.doLocalName(Identifier.ITERATOR2) + case NoRepeat => memberName + } + + out.puts(s"let mut io = Cursor::new($args);") + "io" + } + + override def useIO(ioEx: Ast.expr): String = { + out.puts(s"let mut io = ${expression(ioEx)};") + "io" + } + + override def pushPos(io: String): Unit = + out.puts(s"let _pos = $io.pos();") + + override def seek(io: String, pos: Ast.expr): Unit = + out.puts(s"$io.seek(${expression(pos)});") + + override def popPos(io: String): Unit = + out.puts(s"$io.seek(_pos);") + + override def alignToByte(io: String): Unit = + out.puts(s"$io.alignToByte();") + + override def condIfHeader(expr: Ast.expr): Unit = { + out.puts(s"if ${expression(expr)} {") + out.inc + } + + override def condRepeatEosHeader(id: Identifier, io: String, dataType: DataType, needRaw: Boolean): Unit = { + if (needRaw) + out.puts(s"${privateMemberName(RawIdentifier(id))} = [];") + out.puts(s"${privateMemberName(id)} = [];") + out.puts(s"while !$io.isEof() {") + out.inc + } + + override def handleAssignmentRepeatEos(id: Identifier, expr: String): Unit = { + out.puts(s"${privateMemberName(id)}.push($expr);") + } + + override def condRepeatEosFooter: Unit = { + super.condRepeatEosFooter + } + + override def condRepeatExprHeader(id: Identifier, io: String, dataType: DataType, needRaw: Boolean, repeatExpr: Ast.expr): Unit = { + if (needRaw) + out.puts(s"${privateMemberName(RawIdentifier(id))} = vec!();") + out.puts(s"${privateMemberName(id)} = vec!();") + out.puts(s"for i in 0..${expression(repeatExpr)} {") + out.inc + } + + override def handleAssignmentRepeatExpr(id: Identifier, expr: String): Unit = { + out.puts(s"${privateMemberName(id)}.push($expr);") + } + + override def condRepeatUntilHeader(id: Identifier, io: String, dataType: DataType, needRaw: Boolean, untilExpr: Ast.expr): Unit = { + if (needRaw) + out.puts(s"${privateMemberName(RawIdentifier(id))} = vec!();") + out.puts(s"${privateMemberName(id)} = vec!();") + out.puts("while {") + out.inc + } + + override def handleAssignmentRepeatUntil(id: Identifier, expr: String, isRaw: Boolean): Unit = { + val tempVar = if (isRaw) { + translator.doLocalName(Identifier.ITERATOR2) + } else { + translator.doLocalName(Identifier.ITERATOR) + } + out.puts(s"let $tempVar = $expr;") + out.puts(s"${privateMemberName(id)}.append($expr);") + } + + override def condRepeatUntilFooter(id: Identifier, io: String, dataType: DataType, needRaw: Boolean, untilExpr: Ast.expr): Unit = { + typeProvider._currentIteratorType = Some(dataType) + out.puts(s"!(${expression(untilExpr)})") + out.dec + out.puts("} { }") + } + + override def handleAssignmentSimple(id: Identifier, expr: String): Unit = { + out.puts(s"${privateMemberName(id)} = $expr;") + } + + override def parseExpr(dataType: DataType, assignType: DataType, io: String, defEndian: Option[FixedEndian]): String = { + dataType match { + case t: ReadableType => + s"$io.read_${t.apiCall(defEndian)}()?" + case blt: BytesLimitType => + s"$io.read_bytes(${expression(blt.size)})?" + case _: BytesEosType => + s"$io.read_bytes_full()?" + case BytesTerminatedType(terminator, include, consume, eosError, _) => + s"$io.read_bytes_term($terminator, $include, $consume, $eosError)?" + case BitsType1 => + s"$io.read_bits_int(1)? != 0" + case BitsType(width: Int) => + s"$io.read_bits_int($width)?" + case t: UserType => + val addParams = Utils.join(t.args.map((a) => translator.translate(a)), "", ", ", ", ") + val addArgs = if (t.isOpaque) { + "" + } else { + val parent = t.forcedParent match { + case Some(USER_TYPE_NO_PARENT) => "null" + case Some(fp) => translator.translate(fp) + case None => "self" + } + val addEndian = t.classSpec.get.meta.endian match { + case Some(InheritedEndian) => s", ${privateMemberName(EndianIdentifier)}" + case _ => "" + } + s", $parent, ${privateMemberName(RootIdentifier)}$addEndian" + } + + s"Box::new(${translator.types2classAbs(t.classSpec.get.name)}::new(self.stream, self, _root)?)" + } + } + + override def bytesPadTermExpr(expr0: String, padRight: Option[Int], terminator: Option[Int], include: Boolean): String = { + val expr1 = padRight match { + case Some(padByte) => s"$kstreamName::bytesStripRight($expr0, $padByte)" + case None => expr0 + } + val expr2 = terminator match { + case Some(term) => s"$kstreamName::bytesTerminate($expr1, $term, $include)" + case None => expr1 + } + expr2 + } + + var switchIfs = false + val NAME_SWITCH_ON = Ast.expr.Name(Ast.identifier(Identifier.SWITCH_ON)) + + override def switchStart(id: Identifier, on: Ast.expr): Unit = { + val onType = translator.detectType(on) + + switchIfs = onType match { + case _: ArrayType | _: BytesType => true + case _ => false + } + + if (!switchIfs) { + out.puts(s"match ${expression(on)} {") + out.inc + } + } + + def switchCmpExpr(condition: Ast.expr): String = + expression( + Ast.expr.Compare( + NAME_SWITCH_ON, + Ast.cmpop.Eq, + condition + ) + ) + + override def switchCaseFirstStart(condition: Ast.expr): Unit = { + if (switchIfs) { + out.puts(s"if ${switchCmpExpr(condition)} {") + out.inc + } else { + switchCaseStart(condition) + } + } + + override def switchCaseStart(condition: Ast.expr): Unit = { + if (switchIfs) { + out.puts(s"elss if ${switchCmpExpr(condition)} {") + out.inc + } else { + out.puts(s"${expression(condition)} => {") + out.inc + } + } + + override def switchCaseEnd(): Unit = { + if (switchIfs) { + out.dec + out.puts("}") + } else { + out.dec + out.puts("},") + } + } + + override def switchElseStart(): Unit = { + if (switchIfs) { + out.puts("else {") + out.inc + } else { + out.puts("_ => {") + out.inc + } + } + + override def switchElseEnd(): Unit = { + out.dec + out.puts("}") + } + + override def switchEnd(): Unit = universalFooter + + override def instanceDeclaration(attrName: InstanceIdentifier, attrType: DataType, isNullable: Boolean): Unit = { + out.puts(s" pub ${idToStr(attrName)}: Option<${kaitaiType2NativeType(attrType)}>,") + } + + override def instanceDeclHeader(className: List[String]): Unit = { + out.dec + out.puts("}") + out.puts + + out.puts(s"impl ${type2class(className)} {") + out.inc + } + + override def instanceHeader(className: List[String], instName: InstanceIdentifier, dataType: DataType, isNullable: Boolean): Unit = { + out.puts(s"fn ${idToStr(instName)}(&mut self) -> ${kaitaiType2NativeType(dataType)} {") + out.inc + } + + override def instanceCheckCacheAndReturn(instName: InstanceIdentifier, dataType: DataType): Unit = { + out.puts(s"if let Some(x) = ${privateMemberName(instName)} {") + out.inc + out.puts("return x;") + out.dec + out.puts("}") + out.puts + } + + override def instanceReturn(instName: InstanceIdentifier, attrType: DataType): Unit = { + out.puts(s"return ${privateMemberName(instName)};") + } + + override def enumDeclaration(curClass: List[String], enumName: String, enumColl: Seq[(Long, EnumValueSpec)]): Unit = { + val enumClass = type2class(curClass ::: List(enumName)) + + out.puts(s"enum $enumClass {") + out.inc + + enumColl.foreach { case (id, label) => + universalDoc(label.doc) + out.puts(s"${value2Const(label.name)},") + } + + out.dec + out.puts("}") + } + + def value2Const(label: String) = label.toUpperCase + + def idToStr(id: Identifier): String = { + id match { + case SpecialIdentifier(name) => name + case NamedIdentifier(name) => Utils.lowerCamelCase(name) + case NumberedIdentifier(idx) => s"_${NumberedIdentifier.TEMPLATE}$idx" + case InstanceIdentifier(name) => Utils.lowerCamelCase(name) + case RawIdentifier(innerId) => "_raw_" + idToStr(innerId) + } + } + + override def privateMemberName(id: Identifier): String = { + id match { + case IoIdentifier => s"self.stream" + case RootIdentifier => s"_root" + case ParentIdentifier => s"_parent" + case _ => s"self.${idToStr(id)}" + } + } + + override def publicMemberName(id: Identifier) = idToStr(id) + + override def localTemporaryName(id: Identifier): String = s"$$_t_${idToStr(id)}" + + override def paramName(id: Identifier): String = s"${idToStr(id)}" + + def kaitaiType2NativeType(attrType: DataType): String = { + attrType match { + case Int1Type(false) => "u8" + case IntMultiType(false, Width2, _) => "u16" + case IntMultiType(false, Width4, _) => "u32" + case IntMultiType(false, Width8, _) => "u64" + + case Int1Type(true) => "i8" + case IntMultiType(true, Width2, _) => "i16" + case IntMultiType(true, Width4, _) => "i32" + case IntMultiType(true, Width8, _) => "i64" + + case FloatMultiType(Width4, _) => "f32" + case FloatMultiType(Width8, _) => "f64" + + case BitsType(_) => "u64" + + case _: BooleanType => "bool" + case CalcIntType => "i32" + case CalcFloatType => "f64" + + case _: StrType => "String" + case _: BytesType => "Vec" + + case t: UserType => t.classSpec match { + case Some(cs) => s"Box<${type2class(cs.name)}>" + case None => s"Box<${type2class(t.name)}>" + } + + case t: EnumType => t.enumSpec match { + case Some(cs) => s"Box<${type2class(cs.name)}>" + case None => s"Box<${type2class(t.name)}>" + } + + case ArrayType(inType) => s"Vec<${kaitaiType2NativeType(inType)}>" + + case KaitaiStreamType => s"Option>" + case KaitaiStructType | CalcKaitaiStructType => s"Option>" + + case st: SwitchType => kaitaiType2NativeType(st.combinedType) + } + } + + def kaitaiType2Default(attrType: DataType): String = { + attrType match { + case Int1Type(false) => "0" + case IntMultiType(false, Width2, _) => "0" + case IntMultiType(false, Width4, _) => "0" + case IntMultiType(false, Width8, _) => "0" + + case Int1Type(true) => "0" + case IntMultiType(true, Width2, _) => "0" + case IntMultiType(true, Width4, _) => "0" + case IntMultiType(true, Width8, _) => "0" + + case FloatMultiType(Width4, _) => "0" + case FloatMultiType(Width8, _) => "0" + + case BitsType(_) => "0" + + case _: BooleanType => "false" + case CalcIntType => "0" + case CalcFloatType => "0" + + case _: StrType => "\"\"" + case _: BytesType => "vec!()" + + case t: UserType => "Default::default()" + case t: EnumType => "Default::default()" + + case ArrayType(inType) => "vec!()" + + case KaitaiStreamType => "None" + case KaitaiStructType => "None" + + case _: SwitchType => "" + // TODO + } + } + + def type2class(names: List[String]) = types2classRel(names) + + def type2classAbs(names: List[String]) = + names.mkString("::") + + override def ksErrorName(err: KSError): String = RustCompiler.ksErrorName(err) +} + +object RustCompiler extends LanguageCompilerStatic + with StreamStructNames + with UpperCamelCaseClasses + with ExceptionNames { + override def getCompiler( + tp: ClassTypeProvider, + config: RuntimeConfig + ): LanguageCompiler = new RustCompiler(tp, config) + + override def kstructName = "&Option>" + override def kstreamName = "&mut S" + override def ksErrorName(err: KSError): String = ??? + + def types2class(typeName: Ast.typeId) = { + typeName.names.map(type2class).mkString( + if (typeName.absolute) "__" else "", + "__", + "" + ) + } + + def types2classRel(names: List[String]) = + names.map(type2class).mkString("__") +} diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/AllocateAndStoreIO.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/AllocateAndStoreIO.scala index bb216d6c5..731cfc2ef 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/AllocateAndStoreIO.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/AllocateAndStoreIO.scala @@ -1,12 +1,22 @@ package io.kaitai.struct.languages.components -import io.kaitai.struct.format.{Identifier, IoStorageIdentifier, RepeatSpec} +import io.kaitai.struct.datatype.DataType.{ArrayType, KaitaiStreamType} +import io.kaitai.struct.format._ /** * Allocates new IO and returns attribute identifier that it will be stored * at. This is used for languages without garbage collection that need to * keep track of allocated IOs. */ -trait AllocateAndStoreIO { - def allocateIO(varName: Identifier, rep: RepeatSpec): IoStorageIdentifier +trait AllocateAndStoreIO extends ExtraAttrs { + def allocateIO(id: Identifier, rep: RepeatSpec): String + + override def extraAttrForIO(id: Identifier, rep: RepeatSpec): List[AttrSpec] = { + val ioId = IoStorageIdentifier(id) + val ioType = rep match { + case NoRepeat => KaitaiStreamType + case _ => ArrayType(KaitaiStreamType) + } + List(AttrSpec(List(), ioId, ioType)) + } } diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/AllocateIOLocalVar.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/AllocateIOLocalVar.scala index b7b411c03..32cb79047 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/AllocateIOLocalVar.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/AllocateIOLocalVar.scala @@ -1,12 +1,12 @@ package io.kaitai.struct.languages.components -import io.kaitai.struct.format.{Identifier, RepeatSpec} +import io.kaitai.struct.format.{AttrSpec, Identifier, RepeatSpec} /** * Allocates new auxiliary IOs as local vars - no references saved and thus * probably garbage collector will deal with them. */ -trait AllocateIOLocalVar { +trait AllocateIOLocalVar extends ExtraAttrs { def allocateIO(varName: Identifier, rep: RepeatSpec): String /** @@ -25,4 +25,5 @@ trait AllocateIOLocalVar { * @return name of generated IO local variable as string */ def allocateIOGrowing(varName: Identifier): String = ??? + override def extraAttrForIO(id: Identifier, rep: RepeatSpec): List[AttrSpec] = List() } diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/CommonReads.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/CommonReads.scala new file mode 100644 index 000000000..d938b7b98 --- /dev/null +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/CommonReads.scala @@ -0,0 +1,89 @@ +package io.kaitai.struct.languages.components + +import io.kaitai.struct.datatype._ +import io.kaitai.struct.datatype.DataType.{SwitchType, UserTypeFromBytes} +import io.kaitai.struct.exprlang.Ast +import io.kaitai.struct.format._ + +trait CommonReads extends LanguageCompiler { + override def attrParse(attr: AttrLikeSpec, id: Identifier, defEndian: Option[Endianness]): Unit = { + attrParseIfHeader(id, attr.cond.ifExpr) + + // Manage IO & seeking for ParseInstances + val io = attr match { + case pis: ParseInstanceSpec => + val io = pis.io match { + case None => normalIO + case Some(ex) => useIO(ex) + } + pis.pos.foreach { pos => + pushPos(io) + seek(io, pos) + } + io + case _ => + // no seeking required for sequence attributes + normalIO + } + + if (config.readStoresPos) + attrDebugStart(id, attr.dataType, Some(io), NoRepeat) + + defEndian match { + case Some(_: CalcEndian) | Some(InheritedEndian) => + attrParseHybrid( + () => attrParse0(id, attr, io, Some(LittleEndian)), + () => attrParse0(id, attr, io, Some(BigEndian)) + ) + case None => + attrParse0(id, attr, io, None) + case Some(fe: FixedEndian) => + attrParse0(id, attr, io, Some(fe)) + } + + if (config.readStoresPos) + attrDebugEnd(id, attr.dataType, io, NoRepeat) + + // More position management after parsing for ParseInstanceSpecs + attr match { + case pis: ParseInstanceSpec => + if (pis.pos.isDefined) + popPos(io) + case _ => // no seeking required for sequence attributes + } + + attrParseIfFooter(attr.cond.ifExpr) + } + + def attrParse0(id: Identifier, attr: AttrLikeSpec, io: String, defEndian: Option[FixedEndian]): Unit = { + attr.cond.repeat match { + case RepeatEos => + condRepeatEosHeader(id, io, attr.dataType, needRaw(attr.dataType)) + attrParse2(id, attr.dataType, io, attr.cond.repeat, false, defEndian) + condRepeatEosFooter + case RepeatExpr(repeatExpr: Ast.expr) => + condRepeatExprHeader(id, io, attr.dataType, needRaw(attr.dataType), repeatExpr) + attrParse2(id, attr.dataType, io, attr.cond.repeat, false, defEndian) + condRepeatExprFooter + case RepeatUntil(untilExpr: Ast.expr) => + condRepeatUntilHeader(id, io, attr.dataType, needRaw(attr.dataType), untilExpr) + attrParse2(id, attr.dataType, io, attr.cond.repeat, false, defEndian) + condRepeatUntilFooter(id, io, attr.dataType, needRaw(attr.dataType), untilExpr) + case NoRepeat => + attrParse2(id, attr.dataType, io, attr.cond.repeat, false, defEndian) + } + } + + def attrParse2(id: Identifier, dataType: DataType, io: String, rep: RepeatSpec, isRaw: Boolean, defEndian: Option[FixedEndian], assignType: Option[DataType] = None): Unit + + def needRaw(dataType: DataType): Boolean = { + dataType match { + case _: UserTypeFromBytes => true + case st: SwitchType => st.hasSize + case _ => false + } + } + + def attrDebugStart(attrName: Identifier, attrType: DataType, io: Option[String], repeat: RepeatSpec): Unit = {} + def attrDebugEnd(attrName: Identifier, attrType: DataType, io: String, repeat: RepeatSpec): Unit = {} +} diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/EveryReadIsExpression.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/EveryReadIsExpression.scala index cefb05751..08ee46d99 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/EveryReadIsExpression.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/EveryReadIsExpression.scala @@ -1,10 +1,11 @@ package io.kaitai.struct.languages.components import io.kaitai.struct.Utils -import io.kaitai.struct.exprlang.Ast -import io.kaitai.struct.datatype.DataType import io.kaitai.struct.datatype.DataType._ +import io.kaitai.struct.datatype.{DataType, FixedEndian} +import io.kaitai.struct.exprlang.Ast import io.kaitai.struct.format._ +import io.kaitai.struct.translators.BaseTranslator import scala.collection.mutable.ListBuffer @@ -13,93 +14,54 @@ import scala.collection.mutable.ListBuffer * rvalue. In these languages, "attrStdTypeParse" is replaced with higher-level API: "stdTypeParseExpr" and * "handleAssignment". */ -trait EveryReadIsExpression extends LanguageCompiler with ObjectOrientedLanguage { - override def attrParse(attr: AttrLikeSpec, id: Identifier, extraAttrs: ListBuffer[AttrSpec]): Unit = { - attrParseIfHeader(id, attr.cond.ifExpr) - - // Manage IO & seeking for ParseInstances - val io = attr match { - case pis: ParseInstanceSpec => - val io = pis.io match { - case None => normalIO - case Some(ex) => useIO(ex) - } - pis.pos.foreach { pos => - pushPos(io) - seek(io, pos) - } - io - case _ => - // no seeking required for sequence attributes - normalIO - } - - if (debug) - attrDebugStart(id, attr.dataType, Some(io), NoRepeat) - - attr.cond.repeat match { - case RepeatEos => - condRepeatEosHeader(id, io, attr.dataType, needRaw(attr.dataType)) - attrParse2(id, attr.dataType, io, extraAttrs, attr.cond.repeat, false) - condRepeatEosFooter - case RepeatExpr(repeatExpr: Ast.expr) => - condRepeatExprHeader(id, io, attr.dataType, needRaw(attr.dataType), repeatExpr) - attrParse2(id, attr.dataType, io, extraAttrs, attr.cond.repeat, false) - condRepeatExprFooter - case RepeatUntil(untilExpr: Ast.expr) => - condRepeatUntilHeader(id, io, attr.dataType, needRaw(attr.dataType), untilExpr) - attrParse2(id, attr.dataType, io, extraAttrs, attr.cond.repeat, false) - condRepeatUntilFooter(id, io, attr.dataType, needRaw(attr.dataType), untilExpr) - case NoRepeat => - attrParse2(id, attr.dataType, io, extraAttrs, attr.cond.repeat, false) - } - - if (debug) - attrDebugEnd(id, attr.dataType, io, NoRepeat) - - // More position management after parsing for ParseInstanceSpecs - attr match { - case pis: ParseInstanceSpec => - if (pis.pos.isDefined) - popPos(io) - case _ => // no seeking required for sequence attributes - } - - attrParseIfFooter(attr.cond.ifExpr) - } - - def attrParse2( +trait EveryReadIsExpression + extends LanguageCompiler + with ObjectOrientedLanguage + with CommonReads + with SwitchOps { + val translator: BaseTranslator + + override def attrParse2( id: Identifier, dataType: DataType, io: String, - extraAttrs: ListBuffer[AttrSpec], rep: RepeatSpec, - isRaw: Boolean + isRaw: Boolean, + defEndian: Option[FixedEndian], + assignTypeOpt: Option[DataType] = None ): Unit = { - if (debug && rep != NoRepeat) + val assignType = assignTypeOpt.getOrElse(dataType) + + if (config.readStoresPos && rep != NoRepeat) attrDebugStart(id, dataType, Some(io), rep) dataType match { case FixedBytesType(c, _) => attrFixedContentsParse(id, c) case t: UserType => - attrUserTypeParse(id, t, io, extraAttrs, rep) + attrUserTypeParse(id, t, io, rep, defEndian) case t: BytesType => - attrBytesTypeParse(id, t, io, extraAttrs, rep, isRaw) - case SwitchType(on, cases) => - attrSwitchTypeParse(id, on, cases, io, extraAttrs, rep) + attrBytesTypeParse(id, t, io, rep, isRaw) + case st: SwitchType => + val isNullable = if (switchBytesOnlyAsRaw) { + st.isNullableSwitchRaw + } else { + st.isNullable + } + + attrSwitchTypeParse(id, st.on, st.cases, io, rep, defEndian, isNullable, st.combinedType) case t: StrFromBytesType => val expr = translator.bytesToStr(parseExprBytes(t.bytes, io), Ast.expr.Str(t.encoding)) handleAssignment(id, expr, rep, isRaw) case t: EnumType => - val expr = translator.doEnumById(t.enumSpec.get.name, parseExpr(t.basedOn, io)) + val expr = translator.doEnumById(t.enumSpec.get.name, parseExpr(t.basedOn, t.basedOn, io, defEndian)) handleAssignment(id, expr, rep, isRaw) case _ => - val expr = parseExpr(dataType, io) + val expr = parseExpr(dataType, assignType, io, defEndian) handleAssignment(id, expr, rep, isRaw) } - if (debug && rep != NoRepeat) + if (config.readStoresPos && rep != NoRepeat) attrDebugEnd(id, dataType, io, rep) } @@ -107,17 +69,13 @@ trait EveryReadIsExpression extends LanguageCompiler with ObjectOrientedLanguage id: Identifier, dataType: BytesType, io: String, - extraAttrs: ListBuffer[AttrSpec], rep: RepeatSpec, isRaw: Boolean ): Unit = { // use intermediate variable name, if we'll be doing post-processing val rawId = dataType.process match { case None => id - case Some(_) => - val rawId = RawIdentifier(id) - Utils.addUniqueAttr(extraAttrs, AttrSpec(List(), rawId, dataType)) - rawId + case Some(_) => RawIdentifier(id) } val expr = parseExprBytes(dataType, io) @@ -128,7 +86,7 @@ trait EveryReadIsExpression extends LanguageCompiler with ObjectOrientedLanguage } def parseExprBytes(dataType: BytesType, io: String): String = { - val expr = parseExpr(dataType, io) + val expr = parseExpr(dataType, dataType, io, None) // apply pad stripping and termination dataType match { @@ -141,27 +99,23 @@ trait EveryReadIsExpression extends LanguageCompiler with ObjectOrientedLanguage } } - def attrUserTypeParse(id: Identifier, dataType: UserType, io: String, extraAttrs: ListBuffer[AttrSpec], rep: RepeatSpec): Unit = { + def attrUserTypeParse(id: Identifier, dataType: UserType, io: String, rep: RepeatSpec, defEndian: Option[FixedEndian]): Unit = { val newIO = dataType match { case knownSizeType: UserTypeFromBytes => // we have a fixed buffer, thus we shall create separate IO for it val rawId = RawIdentifier(id) val byteType = knownSizeType.bytes - attrParse2(rawId, byteType, io, extraAttrs, rep, true) + attrParse2(rawId, byteType, io, rep, true, defEndian) val extraType = rep match { case NoRepeat => byteType case _ => ArrayType(byteType) } - Utils.addUniqueAttr(extraAttrs, AttrSpec(List(), rawId, extraType)) - this match { case thisStore: AllocateAndStoreIO => - val ourIO = thisStore.allocateIO(rawId, rep) - Utils.addUniqueAttr(extraAttrs, AttrSpec(List(), ourIO, KaitaiStreamType)) - privateMemberName(ourIO) + thisStore.allocateIO(rawId, rep) case thisLocal: AllocateIOLocalVar => thisLocal.allocateIO(rawId, rep) } @@ -169,20 +123,20 @@ trait EveryReadIsExpression extends LanguageCompiler with ObjectOrientedLanguage // no fixed buffer, just use regular IO io } - val expr = parseExpr(dataType, newIO) - if (!debug) { + val expr = parseExpr(dataType, dataType, newIO, defEndian) + if (config.autoRead) { handleAssignment(id, expr, rep, false) } else { - // Debug mode requires one to actually call "_read" method on constructed user type, - // and this must be done as a separate statement - or else exception handler would - // blast the whole structure, not only this element. This, in turn, makes us assign - // constructed element to a temporary variable in case on repetitions + // Disabled autoRead requires one to actually call `_read` method on constructed + // user type, and this must be done as a separate statement - or else exception + // handler would blast the whole structure, not only this element. This, in turn, + // makes us assign constructed element to a temporary variable in case of arrays. rep match { case NoRepeat => handleAssignmentSimple(id, expr) userTypeDebugRead(privateMemberName(id)) case _ => - val tempVarName = s"_t_${idToStr(id)}" + val tempVarName = localTemporaryName(id) handleAssignmentTempVar(dataType, tempVarName, expr) userTypeDebugRead(tempVarName) handleAssignment(id, tempVarName, rep, false) @@ -190,57 +144,36 @@ trait EveryReadIsExpression extends LanguageCompiler with ObjectOrientedLanguage } } - def needRaw(dataType: DataType): Boolean = { - dataType match { - case t: UserTypeFromBytes => true - case _ => false - } - } - - def attrSwitchTypeParse(id: Identifier, on: Ast.expr, cases: Map[Ast.expr, DataType], io: String, extraAttrs: ListBuffer[AttrSpec], rep: RepeatSpec): Unit = { - switchStart(id, on) - - // Pass 1: only normal case clauses - var first = true - - cases.foreach { case (condition, dataType) => - condition match { - case SwitchType.ELSE_CONST => - // skip for now - case _ => - if (first) { - switchCaseFirstStart(condition) - first = false - } else { - switchCaseStart(condition) - } - attrParse2(id, dataType, io, extraAttrs, rep, false) - switchCaseEnd() - } - } - - // Pass 2: else clause, if it is there - cases.foreach { case (condition, dataType) => - condition match { - case SwitchType.ELSE_CONST => - switchElseStart() - if (switchBytesOnlyAsRaw) { - dataType match { - case t: BytesType => - attrParse2(RawIdentifier(id), dataType, io, extraAttrs, rep, false) - case _ => - attrParse2(id, dataType, io, extraAttrs, rep, false) - } - } else { - attrParse2(id, dataType, io, extraAttrs, rep, false) - } - switchElseEnd() - case _ => - // ignore normal case clauses + def attrSwitchTypeParse( + id: Identifier, + on: Ast.expr, + cases: Map[Ast.expr, DataType], + io: String, + rep: RepeatSpec, + defEndian: Option[FixedEndian], + isNullable: Boolean, + assignType: DataType + ): Unit = { + if (isNullable) + condIfSetNull(id) + + switchCases[DataType](id, on, cases, + (dataType) => { + if (isNullable) + condIfSetNonNull(id) + attrParse2(id, dataType, io, rep, false, defEndian, Some(assignType)) + }, + (dataType) => if (switchBytesOnlyAsRaw) { + dataType match { + case t: BytesType => + attrParse2(RawIdentifier(id), dataType, io, rep, false, defEndian, Some(assignType)) + case _ => + attrParse2(id, dataType, io, rep, false, defEndian, Some(assignType)) + } + } else { + attrParse2(id, dataType, io, rep, false, defEndian, Some(assignType)) } - } - - switchEnd() + ) } def handleAssignment(id: Identifier, expr: String, rep: RepeatSpec, isRaw: Boolean): Unit = { @@ -252,40 +185,19 @@ trait EveryReadIsExpression extends LanguageCompiler with ObjectOrientedLanguage } } - def attrDebugStart(attrName: Identifier, attrType: DataType, io: Option[String], repeat: RepeatSpec): Unit = {} - def attrDebugEnd(attrName: Identifier, attrType: DataType, io: String, repeat: RepeatSpec): Unit = {} - def handleAssignmentRepeatEos(id: Identifier, expr: String): Unit def handleAssignmentRepeatExpr(id: Identifier, expr: String): Unit def handleAssignmentRepeatUntil(id: Identifier, expr: String, isRaw: Boolean): Unit def handleAssignmentSimple(id: Identifier, expr: String): Unit def handleAssignmentTempVar(dataType: DataType, id: String, expr: String): Unit = ??? - def parseExpr(dataType: DataType, io: String): String + def parseExpr(dataType: DataType, assignType: DataType, io: String, defEndian: Option[FixedEndian]): String def bytesPadTermExpr(expr0: String, padRight: Option[Int], terminator: Option[Int], include: Boolean): String - def userTypeDebugRead(id: String): Unit = {} + def userTypeDebugRead(id: String): Unit = ??? - def instanceCalculate(instName: InstanceIdentifier, dataType: DataType, value: Ast.expr): Unit = { - if (debug) + def instanceCalculate(instName: Identifier, dataType: DataType, value: Ast.expr): Unit = { + if (config.readStoresPos) attrDebugStart(instName, dataType, None, NoRepeat) handleAssignmentSimple(instName, expression(value)) } - - def switchStart(id: Identifier, on: Ast.expr): Unit - def switchCaseFirstStart(condition: Ast.expr): Unit = switchCaseStart(condition) - def switchCaseStart(condition: Ast.expr): Unit - def switchCaseEnd(): Unit - def switchElseStart(): Unit - def switchElseEnd(): Unit = switchCaseEnd() - def switchEnd(): Unit - - /** - * Controls parsing of typeless (BytesType) alternative in switch case. If true, - * then target language does not support storing both bytes array and true object - * in the same variable, so we'll use workaround: bytes array will be read as - * _raw_ variable (which would be used anyway for all other cases as well). If - * false (which is default), we'll store *both* true objects and bytes array in - * the same variable. - */ - def switchBytesOnlyAsRaw = false } diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala index f8f15a75d..f336e8ca2 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala @@ -2,6 +2,7 @@ package io.kaitai.struct.languages.components import io.kaitai.struct.datatype.DataType import io.kaitai.struct.datatype.DataType._ +import io.kaitai.struct.datatype.FixedEndian import io.kaitai.struct.exprlang.Ast import io.kaitai.struct.exprlang.Ast.expr import io.kaitai.struct.format._ @@ -9,7 +10,7 @@ import io.kaitai.struct.format._ import scala.collection.mutable.ListBuffer trait EveryWriteIsExpression extends LanguageCompiler with ObjectOrientedLanguage with EveryReadIsExpression { - override def attrWrite(attr: AttrLikeSpec, id: Identifier, extraAttrs: ListBuffer[AttrSpec]): Unit = { + override def attrWrite(attr: AttrLikeSpec, id: Identifier, defEndian: Option[FixedEndian]): Unit = { attrParseIfHeader(id, attr.cond.ifExpr) val io = normalIO @@ -17,18 +18,18 @@ trait EveryWriteIsExpression extends LanguageCompiler with ObjectOrientedLanguag attr.cond.repeat match { case RepeatEos => condRepeatEosHeader2(id, io, attr.dataType, needRaww(attr.dataType)) - attrWrite2(id, attr.dataType, io, extraAttrs, attr.cond.repeat, false) + attrWrite2(id, attr.dataType, io, defEndian, attr.cond.repeat, false) condRepeatEosFooter case RepeatExpr(repeatExpr: Ast.expr) => condRepeatExprHeader2(id, io, attr.dataType, needRaww(attr.dataType), repeatExpr) - attrWrite2(id, attr.dataType, io, extraAttrs, attr.cond.repeat, false) + attrWrite2(id, attr.dataType, io, defEndian, attr.cond.repeat, false) condRepeatExprFooter case RepeatUntil(untilExpr: Ast.expr) => condRepeatUntilHeader(id, io, attr.dataType, needRaww(attr.dataType), untilExpr) - attrWrite2(id, attr.dataType, io, extraAttrs, attr.cond.repeat, false) + attrWrite2(id, attr.dataType, io, defEndian, attr.cond.repeat, false) condRepeatUntilFooter(id, io, attr.dataType, needRaww(attr.dataType), untilExpr) case NoRepeat => - attrWrite2(id, attr.dataType, io, extraAttrs, attr.cond.repeat, false) + attrWrite2(id, attr.dataType, io, defEndian, attr.cond.repeat, false) } attrParseIfFooter(attr.cond.ifExpr) @@ -38,43 +39,42 @@ trait EveryWriteIsExpression extends LanguageCompiler with ObjectOrientedLanguag id: Identifier, dataType: DataType, io: String, - extraAttrs: ListBuffer[AttrSpec], + defEndian: Option[FixedEndian], rep: RepeatSpec, isRaw: Boolean ): Unit = { dataType match { case t: UserType => - attrUserTypeWrite(id, t, io, extraAttrs, rep, isRaw) + attrUserTypeWrite(id, t, io, rep, isRaw, defEndian) case t: BytesType => - attrBytesTypeWrite(id, t, io, extraAttrs, rep, isRaw) - case SwitchType(on, cases) => - attrSwitchTypeWrite(id, on, cases, io, extraAttrs, rep) + attrBytesTypeWrite(id, t, io, rep, isRaw) + case SwitchType(on, cases, _) => + attrSwitchTypeWrite(id, on, cases, io, rep, defEndian) case t: StrFromBytesType => - attrStrTypeWrite(id, t, io, extraAttrs, rep, isRaw) + attrStrTypeWrite(id, t, io, rep, isRaw) case t: EnumType => val expr = translator.enumToInt(Ast.expr.Name(Ast.identifier(idToStr(id))), t) - attrPrimitiveWrite(io, expr, t.basedOn) + attrPrimitiveWrite(io, expr, t.basedOn, defEndian) case _ => val expr = writeExpr(id, rep, isRaw) - attrPrimitiveWrite(io, expr, dataType) + attrPrimitiveWrite(io, translator.translate(expr), dataType, defEndian) } } - def writeExpr(id: Identifier, rep: RepeatSpec, isRaw: Boolean): String = { + def writeExpr(id: Identifier, rep: RepeatSpec, isRaw: Boolean): Ast.expr = { + val astId = Ast.expr.Name(Ast.identifier(idToStr(id))) rep match { case _: RepeatExpr | RepeatEos => - translator.translate( - Ast.expr.Subscript( - Ast.expr.Name(Ast.identifier(idToStr(id))), - Ast.expr.Name(Ast.identifier(Identifier.ITERATOR_I)) - ) + Ast.expr.Subscript( + astId, + Ast.expr.Name(Ast.identifier(Identifier.INDEX)) ) case NoRepeat => - privateMemberName(id) + astId } } - def attrBytesTypeWrite(id: Identifier, t: BytesType, io: String, extraAttrs: ListBuffer[AttrSpec], rep: RepeatSpec, isRaw: Boolean) = { + def attrBytesTypeWrite(id: Identifier, t: BytesType, io: String, rep: RepeatSpec, isRaw: Boolean) = { val idToWrite = t.process match { case Some(proc) => val rawId = RawIdentifier(id) @@ -85,40 +85,40 @@ trait EveryWriteIsExpression extends LanguageCompiler with ObjectOrientedLanguag } t match { case FixedBytesType(contents, process) => - attrPrimitiveWrite(io, translator.doByteArrayLiteral(contents), t) + attrPrimitiveWrite(io, translator.doByteArrayLiteral(contents), t, None) case t: BytesEosType => val expr = writeExpr(idToWrite, rep, isRaw) - attrPrimitiveWrite(io, expr, t) + attrPrimitiveWrite(io, translator.translate(expr), t, None) if (t.terminator.isDefined && !t.include) - attrPrimitiveWrite(io, t.terminator.toString, Int1Type(false)) + attrPrimitiveWrite(io, t.terminator.toString, Int1Type(false), None) case blt: BytesLimitType => val expr = writeExpr(idToWrite, rep, isRaw) - attrBytesLimitWrite2(io, expr, blt) + attrBytesLimitWrite2(io, translator.translate(expr), blt) case t: BytesTerminatedType => val expr = writeExpr(idToWrite, rep, isRaw) - attrPrimitiveWrite(io, expr, t) + attrPrimitiveWrite(io, translator.translate(expr), t, None) if (t.consume && !t.include) - attrPrimitiveWrite(io, t.terminator.toString, Int1Type(false)) + attrPrimitiveWrite(io, t.terminator.toString, Int1Type(false), None) } } - def attrStrTypeWrite(id: Identifier, t: StrFromBytesType, io: String, extraAttrs: ListBuffer[AttrSpec], rep: RepeatSpec, isRaw: Boolean) = { + def attrStrTypeWrite(id: Identifier, t: StrFromBytesType, io: String, rep: RepeatSpec, isRaw: Boolean) = { val expr = translator.strToBytes(writeExpr(id, rep, isRaw), Ast.expr.Str(t.encoding)) - attrPrimitiveWrite(io, expr, t.bytes) + attrPrimitiveWrite(io, expr, t.bytes, None) t.bytes match { case t: BytesEosType => if (t.terminator.isDefined && !t.include) - attrPrimitiveWrite(io, t.terminator.toString, Int1Type(false)) + attrPrimitiveWrite(io, t.terminator.toString, Int1Type(false), None) case t: BytesLimitType => // FIXME: implement padding and terminator byte t.terminator.foreach((terminator) => if (!t.include) - attrPrimitiveWrite(io, terminator.toString, Int1Type(false)) + attrPrimitiveWrite(io, terminator.toString, Int1Type(false), None) ) case t: BytesTerminatedType => if (t.consume && !t.include) - attrPrimitiveWrite(io, t.terminator.toString, Int1Type(false)) + attrPrimitiveWrite(io, t.terminator.toString, Int1Type(false), None) } } @@ -127,7 +127,7 @@ trait EveryWriteIsExpression extends LanguageCompiler with ObjectOrientedLanguag case (None, None, false) => // no terminator, no padding => just a regular output // validation should check that expression's length matches size - attrPrimitiveWrite(io, expr, blt) + attrPrimitiveWrite(io, expr, blt, None) return case (_, None, true) => // terminator included, no padding => pad with zeroes @@ -153,15 +153,15 @@ trait EveryWriteIsExpression extends LanguageCompiler with ObjectOrientedLanguag id: Identifier, t: UserType, io: String, - extraAttrs: ListBuffer[AttrSpec], rep: RepeatSpec, - isRaw: Boolean + isRaw: Boolean, + defEndian: Option[FixedEndian] ) = { val expr = writeExpr(id, rep, isRaw) t match { case _: UserTypeInstream => - attrUserTypeInstreamWrite(io, expr) + attrUserTypeInstreamWrite(io, translator.translate(expr)) case knownSizeType: UserTypeFromBytes => val rawId = RawIdentifier(id) val byteType = knownSizeType.bytes @@ -176,11 +176,11 @@ trait EveryWriteIsExpression extends LanguageCompiler with ObjectOrientedLanguag // privateMemberName(ourIO) case thisLocal: AllocateIOLocalVar => val ioFixed = thisLocal.allocateIOFixed(rawId, translator.translate(blt.size)) - attrUserTypeInstreamWrite(ioFixed, expr) + attrUserTypeInstreamWrite(ioFixed, translator.translate(expr)) attrWriteStreamToStream(ioFixed, io) } case _ => - attrUserTypeInstreamWrite(io, expr) + attrUserTypeInstreamWrite(io, translator.translate(expr)) } case Some(process) => byteType match { @@ -188,16 +188,16 @@ trait EveryWriteIsExpression extends LanguageCompiler with ObjectOrientedLanguag this match { case thisLocal: AllocateIOLocalVar => val ioFixed = thisLocal.allocateIOFixed(rawId, translator.translate(blt.size)) - attrUserTypeInstreamWrite(ioFixed, expr) + attrUserTypeInstreamWrite(ioFixed, translator.translate(expr)) handleAssignment(rawId, exprStreamToByteArray(ioFixed), rep, isRaw) - attrBytesTypeWrite(rawId, byteType, io, extraAttrs, rep, isRaw) + attrBytesTypeWrite(rawId, byteType, io, rep, isRaw) } } } } } - def attrSwitchTypeWrite(id: Identifier, on: expr, cases: Map[expr, DataType], io: String, extraAttrs: ListBuffer[AttrSpec], rep: RepeatSpec) = { + def attrSwitchTypeWrite(id: Identifier, on: expr, cases: Map[expr, DataType], io: String, rep: RepeatSpec, defEndian: Option[FixedEndian]) = { switchStart(id, on) // Pass 1: only normal case clauses @@ -214,7 +214,7 @@ trait EveryWriteIsExpression extends LanguageCompiler with ObjectOrientedLanguag } else { switchCaseStart(condition) } - attrWrite2(id, dataType, io, extraAttrs, rep, false) + attrWrite2(id, dataType, io, defEndian, rep, false) switchCaseEnd() } } @@ -227,12 +227,12 @@ trait EveryWriteIsExpression extends LanguageCompiler with ObjectOrientedLanguag if (switchBytesOnlyAsRaw) { dataType match { case t: BytesType => - attrWrite2(RawIdentifier(id), dataType, io, extraAttrs, rep, false) + attrWrite2(RawIdentifier(id), dataType, io, defEndian, rep, false) case _ => - attrWrite2(id, dataType, io, extraAttrs, rep, false) + attrWrite2(id, dataType, io, defEndian, rep, false) } } else { - attrWrite2(id, dataType, io, extraAttrs, rep, false) + attrWrite2(id, dataType, io, defEndian, rep, false) } switchElseEnd() case _ => @@ -243,7 +243,7 @@ trait EveryWriteIsExpression extends LanguageCompiler with ObjectOrientedLanguag switchEnd() } - def attrPrimitiveWrite(io: String, expr: String, dt: DataType): Unit + def attrPrimitiveWrite(io: String, expr: String, dt: DataType, defEndian: Option[FixedEndian]): Unit def attrBytesLimitWrite(io: String, expr: String, size: String, term: Int, padRight: Int): Unit def attrUserTypeInstreamWrite(io: String, expr: String): Unit def attrWriteStreamToStream(srcIo: String, dstIo: String): Unit diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/ExceptionNames.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/ExceptionNames.scala new file mode 100644 index 000000000..b73cde71e --- /dev/null +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/ExceptionNames.scala @@ -0,0 +1,18 @@ +package io.kaitai.struct.languages.components + +import io.kaitai.struct.datatype.KSError + +/** + * Stores per-language knowledge on how a particular KS-generated + * runtime exceptions will be named in particular language. + */ +trait ExceptionNames { + /** + * Resolves string name of exception in target language. + * Suggested implementation is to use `err.name` that provides + * UpperCamelCase renditions of original names. + * @param err KS-generated error that might be thrown in runtime + * @return name of exception as a string + */ + def ksErrorName(err: KSError): String +} diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/ExtraAttrs.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/ExtraAttrs.scala new file mode 100644 index 000000000..90f19d3a9 --- /dev/null +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/ExtraAttrs.scala @@ -0,0 +1,66 @@ +package io.kaitai.struct.languages.components + +import io.kaitai.struct.datatype.DataType +import io.kaitai.struct.datatype.DataType._ +import io.kaitai.struct.format._ + +/** + * Trait to be implemented by all [[LanguageCompiler]] compilers: supplies extra attributes + * when we'll be allocating new IOs. + */ +trait ExtraAttrs { + def extraAttrForIO(id: Identifier, rep: RepeatSpec): List[AttrSpec] +} + +/** + * Generates list of extra attributes required to store intermediate / + * virtual stuff for every attribute like: + * + * * buffered raw value byte arrays + * * IO objects (?) + * * unprocessed / postprocessed byte arrays + */ +object ExtraAttrs { + def forClassSpec(curClass: ClassSpec, compiler: ExtraAttrs): List[AttrSpec] = { + // We want only values of ParseInstances, which are AttrSpecLike. + // ValueInstances are ignored, as they can't currently generate + // any extra attributes (i.e. no `size`, no `process`, etc) + val parseInstances = curClass.instances.values.collect { + case inst: AttrLikeSpec => inst + } + + (curClass.seq ++ parseInstances).foldLeft(List[AttrSpec]())( + (attrs, attr) => attrs ++ ExtraAttrs.forAttr(attr, compiler) + ) + } + + def forAttr(attr: AttrLikeSpec, compiler: ExtraAttrs): Iterable[AttrSpec] = + forAttr(attr.id, attr.dataType, attr.cond, compiler) + + private + def forAttr(id: Identifier, dataType: DataType, condSpec: ConditionalSpec, compiler: ExtraAttrs): Iterable[AttrSpec] = { + dataType match { + case bt: BytesType => + // Byte array: only need extra attrs if `process` is used + bt.process match { + case None => List() + case Some(_) => + val rawId = RawIdentifier(id) + List(AttrSpec(List(), rawId, bt, condSpec)) ++ + compiler.extraAttrForIO(id, condSpec.repeat) + } + case utb: UserTypeFromBytes => + // User type in a substream + val rawId = RawIdentifier(id) + (List(AttrSpec(List(), rawId, utb.bytes, condSpec)) ++ + compiler.extraAttrForIO(rawId, condSpec.repeat) ++ + forAttr(rawId, utb.bytes, condSpec, compiler)).toList.distinct + case st: SwitchType => + st.cases.flatMap { case (_, caseType) => + forAttr(id, caseType, condSpec, compiler) + }.toList.distinct + case _ => + List() + } + } +} diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/FixedContentsUsingArrayByteLiteral.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/FixedContentsUsingArrayByteLiteral.scala index b2ea21b50..c5d4abd9b 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/FixedContentsUsingArrayByteLiteral.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/FixedContentsUsingArrayByteLiteral.scala @@ -1,5 +1,6 @@ package io.kaitai.struct.languages.components +import io.kaitai.struct.exprlang.Ast import io.kaitai.struct.format.Identifier /** @@ -8,6 +9,13 @@ import io.kaitai.struct.format.Identifier */ trait FixedContentsUsingArrayByteLiteral extends LanguageCompiler { def attrFixedContentsParse(attrName: Identifier, contents: Array[Byte]) = - attrFixedContentsParse(attrName, translator.doByteArrayLiteral(contents)) + attrFixedContentsParse( + attrName, + translator.translate( + Ast.expr.List( + contents.map(x => Ast.expr.IntNum(BigInt(x & 0xff))) + ) + ) + ) def attrFixedContentsParse(attrName: Identifier, contents: String): Unit } diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/GoReads.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/GoReads.scala new file mode 100644 index 000000000..4089505b4 --- /dev/null +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/GoReads.scala @@ -0,0 +1,158 @@ +package io.kaitai.struct.languages.components + +import io.kaitai.struct.datatype.DataType._ +import io.kaitai.struct.datatype.{DataType, FixedEndian} +import io.kaitai.struct.exprlang.Ast +import io.kaitai.struct.format._ +import io.kaitai.struct.translators.{GoTranslator, ResultLocalVar, ResultString, TranslatorResult} + +trait GoReads extends CommonReads with ObjectOrientedLanguage with GoSwitchOps { + val translator: GoTranslator + + def attrBytesTypeParse( + id: Identifier, + dataType: BytesType, + io: String, + rep: RepeatSpec, + isRaw: Boolean + ): Unit = { + val rawId = dataType.process match { + case None => id + case Some(_) => RawIdentifier(id) + } + val expr = parseExprBytes(translator.outVarCheckRes(parseExpr(dataType, io, None)), dataType) + handleAssignment(rawId, expr, rep, isRaw) + dataType.process.foreach((proc) => attrProcess(proc, rawId, id)) + } + + def attrSwitchTypeParse( + id: Identifier, + on: Ast.expr, + cases: Map[Ast.expr, DataType], + io: String, + rep: RepeatSpec, + defEndian: Option[FixedEndian], + isNullable: Boolean, + assignType: DataType + ): Unit = { + switchCases[DataType](id, on, cases, + (dataType) => { + attrParse2(id, dataType, io, rep, false, defEndian, Some(assignType)) + }, + { + case dataType@(t: BytesType) => + attrParse2(RawIdentifier(id), dataType, io, rep, false, defEndian, Some(assignType)) + case dataType => + attrParse2(id, dataType, io, rep, false, defEndian, Some(assignType)) + } + ) + } + + override def attrParse2( + id: Identifier, + dataType: DataType, + io: String, + rep: RepeatSpec, + isRaw: Boolean, + defEndian: Option[FixedEndian], + assignType: Option[DataType] = None + ): Unit = { + dataType match { + case FixedBytesType(c, _) => + attrFixedContentsParse(id, c) + case t: UserType => + attrUserTypeParse(id, t, io, rep, defEndian) + case t: BytesType => + attrBytesTypeParse(id, t, io, rep, isRaw) + case st: SwitchType => + attrSwitchTypeParse(id, st.on, st.cases, io, rep, defEndian, st.isNullableSwitchRaw, st.combinedType) + case t: StrFromBytesType => + val r1 = parseExprBytes(translator.outVarCheckRes(parseExpr(t.bytes, io, defEndian)), t.bytes) + val expr = translator.bytesToStr(translator.resToStr(r1), Ast.expr.Str(t.encoding)) + handleAssignment(id, expr, rep, isRaw) + case t: EnumType => + val r1 = translator.outVarCheckRes(parseExpr(t.basedOn, io, defEndian)) + val enumSpec = t.enumSpec.get + val expr = translator.trEnumById(enumSpec.name, translator.resToStr(r1)) + handleAssignment(id, expr, rep, isRaw) + case BitsType1 => + val expr = parseExpr(dataType, io, defEndian) + val r1 = translator.outVarCheckRes(expr) + val r2 = ResultString(s"${translator.resToStr(r1)} != 0") + handleAssignment(id, r2, rep, isRaw) + case _ => + val expr = parseExpr(dataType, io, defEndian) + val r = translator.outVarCheckRes(expr) + handleAssignment(id, r, rep, isRaw) + } + } + + def bytesPadTermExpr(id: ResultLocalVar, padRight: Option[Int], terminator: Option[Int], include: Boolean): String = { + val expr0 = translator.resToStr(id) + val expr1 = padRight match { + case Some(padByte) => s"kaitai.BytesStripRight($expr0, $padByte)" + case None => expr0 + } + val expr2 = terminator match { + case Some(term) => s"kaitai.BytesTerminate($expr1, $term, $include)" + case None => expr1 + } + expr2 + } + + def parseExprBytes(id: ResultLocalVar, dataType: BytesType): ResultLocalVar = { + dataType match { + case BytesEosType(terminator, include, padRight, _) => + translator.outTransform(id, bytesPadTermExpr(id, padRight, terminator, include)) + case BytesLimitType(_, terminator, include, padRight, _) => + translator.outTransform(id, bytesPadTermExpr(id, padRight, terminator, include)) + case _ => + id + } + } + + def attrUserTypeParse(id: Identifier, dataType: UserType, io: String, rep: RepeatSpec, defEndian: Option[FixedEndian]): Unit = { + val newIO = dataType match { + case knownSizeType: UserTypeFromBytes => + // we have a fixed buffer, thus we shall create separate IO for it + val rawId = RawIdentifier(id) + val byteType = knownSizeType.bytes + + attrParse2(rawId, byteType, io, rep, true, defEndian) + + val extraType = rep match { + case NoRepeat => byteType + case _ => ArrayType(byteType) + } + + this match { + case thisStore: AllocateAndStoreIO => + thisStore.allocateIO(rawId, rep) + case thisLocal: AllocateIOLocalVar => + thisLocal.allocateIO(rawId, rep) + } + case _: UserTypeInstream => + // no fixed buffer, just use regular IO + io + } + + val expr = translator.userType(dataType, newIO) + handleAssignment(id, expr, rep, false) + } + + def handleAssignment(id: Identifier, expr: TranslatorResult, rep: RepeatSpec, isRaw: Boolean): Unit = { + rep match { + case RepeatEos => handleAssignmentRepeatEos(id, expr) + case RepeatExpr(_) => handleAssignmentRepeatExpr(id, expr) + case RepeatUntil(_) => handleAssignmentRepeatUntil(id, expr, isRaw) + case NoRepeat => handleAssignmentSimple(id, expr) + } + } + + def handleAssignmentRepeatEos(id: Identifier, expr: TranslatorResult): Unit + def handleAssignmentRepeatExpr(id: Identifier, expr: TranslatorResult): Unit + def handleAssignmentRepeatUntil(id: Identifier, expr: TranslatorResult, isRaw: Boolean): Unit + def handleAssignmentSimple(id: Identifier, expr: TranslatorResult): Unit + + def parseExpr(dataType: DataType, io: String, defEndian: Option[FixedEndian]): String +} diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/GoSwitchOps.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/GoSwitchOps.scala new file mode 100644 index 000000000..18ab32ffc --- /dev/null +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/GoSwitchOps.scala @@ -0,0 +1,59 @@ +package io.kaitai.struct.languages.components + +import io.kaitai.struct.datatype.DataType +import io.kaitai.struct.datatype.DataType.{BytesType, SwitchType} +import io.kaitai.struct.exprlang.Ast +import io.kaitai.struct.format.Identifier +import io.kaitai.struct.translators.GoTranslator + +trait GoSwitchOps extends SwitchOps { + val translator: GoTranslator + + def switchShouldUseCompareFn(onType: DataType): Option[String] + def switchCaseStartCompareFn(compareFn: String, switchOn: Ast.expr, condition: Ast.expr): Unit + + override def switchCases[T]( + id: Identifier, + on: Ast.expr, + cases: Map[Ast.expr, T], + normalCaseProc: (T) => Unit, + elseCaseProc: (T) => Unit + ): Unit = { + val onType = translator.detectType(on) + switchShouldUseCompareFn(onType) match { + case Some(compareFn: String) => + switchCasesUsingCompareFn(id, on, compareFn, cases, normalCaseProc, elseCaseProc) + case None => + switchCasesRender(id, on, cases, normalCaseProc, elseCaseProc) + } + } + + protected def switchCasesUsingCompareFn[T]( + id: Identifier, + on: Ast.expr, + compareFn: String, + cases: Map[Ast.expr, T], + normalCaseProc: (T) => Unit, + elseCaseProc: (T) => Unit + ): Unit = { + switchStart(id, Ast.expr.Bool(true)) + + cases.foreach { case (condition, result) => + condition match { + case SwitchType.ELSE_CONST => + case _ => + switchCaseStartCompareFn(compareFn, on, condition) + normalCaseProc(result) + switchCaseEnd() + } + } + + cases.get(SwitchType.ELSE_CONST).foreach { (result) => + switchElseStart() + elseCaseProc(result) + switchElseEnd() + } + + switchEnd() + } +} diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala index aefcefc5a..74c89d6d0 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala @@ -1,18 +1,20 @@ package io.kaitai.struct.languages.components -import io.kaitai.struct.datatype.DataType +import io.kaitai.struct.datatype.{DataType, Endianness, FixedEndian, InheritedEndian} import io.kaitai.struct.exprlang.Ast import io.kaitai.struct.format._ -import io.kaitai.struct.translators.BaseTranslator +import io.kaitai.struct.translators.AbstractTranslator import io.kaitai.struct.{ClassTypeProvider, RuntimeConfig} import scala.collection.mutable.ListBuffer abstract class LanguageCompiler( typeProvider: ClassTypeProvider, - config: RuntimeConfig -) { - val translator: BaseTranslator + val config: RuntimeConfig +) extends SwitchOps with ValidateOps + with ExtraAttrs { + + val translator: AbstractTranslator /** * @return compilation results as a map: keys are file names, values are @@ -39,7 +41,18 @@ abstract class LanguageCompiler( */ def innerEnums: Boolean = true - def debug = config.debug + /** + * Determines whether the language needs docstrings to be generated + * inside classes and methods (true, Python-style) or outside them + * (false, JavaDoc-style, majority of other languages). Affects calling + * sequence of rendering methods. + * + * @return true if language needs docstrings to be generated + * inside classes and methods, false otherwise + */ + def innerDocstrings: Boolean = false + + def debug: Boolean = !config.autoRead && config.readStoresPos def indent: String def outFileName(topClassName: String): String @@ -61,25 +74,33 @@ abstract class LanguageCompiler( def classFooter(name: List[String]): Unit def classForwardDeclaration(name: List[String]): Unit = {} - def classConstructorHeader(name: List[String], parentClassName: List[String], rootClassName: List[String]): Unit + def classConstructorHeader(name: List[String], parentType: DataType, rootClassName: List[String], isHybrid: Boolean, params: List[ParamDefSpec]): Unit def classConstructorFooter: Unit - def classDestructorHeader(name: List[String], parentTypeName: List[String], topClassName: List[String]): Unit = {} + def classDestructorHeader(name: List[String], parentType: DataType, topClassName: List[String]): Unit = {} def classDestructorFooter: Unit = {} - def attributeDeclaration(attrName: Identifier, attrType: DataType, condSpec: ConditionalSpec): Unit - def attributeReader(attrName: Identifier, attrType: DataType, condSpec: ConditionalSpec): Unit + def runRead(): Unit + def runReadCalc(): Unit + def readHeader(endian: Option[FixedEndian], isEmpty: Boolean): Unit + def readFooter(): Unit + + def attributeDeclaration(attrName: Identifier, attrType: DataType, isNullable: Boolean): Unit + def attributeReader(attrName: Identifier, attrType: DataType, isNullable: Boolean): Unit def attributeDoc(id: Identifier, doc: DocSpec): Unit = {} - def attrParse(attr: AttrLikeSpec, id: Identifier, extraAttrs: ListBuffer[AttrSpec]): Unit + def attrParse(attr: AttrLikeSpec, id: Identifier, defEndian: Option[Endianness]): Unit + def attrParseHybrid(leProc: () => Unit, beProc: () => Unit): Unit + def attrInit(attr: AttrLikeSpec): Unit = {} def attrDestructor(attr: AttrLikeSpec, id: Identifier): Unit = {} - def funcWriteHeader(curClass: ClassSpec): Unit = ??? - def funcWriteFooter(curClass: ClassSpec): Unit = ??? - def attrWrite(attr: AttrLikeSpec, id: Identifier, extraAttrs: ListBuffer[AttrSpec]): Unit = ??? + def writeHeader(endian: Option[FixedEndian]): Unit = ??? + def writeFooter(): Unit = ??? + def attrWrite(attr: AttrLikeSpec, id: Identifier, defEndian: Option[FixedEndian]): Unit = ??? + def runWriteCalc(): Unit = ??? - def funcCheckHeader(curClass: ClassSpec): Unit = ??? - def funcCheckFooter(curClass: ClassSpec): Unit = ??? + def checkHeader(): Unit = ??? + def checkFooter(): Unit = ??? def attrCheck(attr: AttrLikeSpec, id: Identifier): Unit = ??? def attrFixedContentsParse(attrName: Identifier, contents: Array[Byte]): Unit @@ -107,16 +128,17 @@ abstract class LanguageCompiler( def popPos(io: String): Unit def alignToByte(io: String): Unit + def instanceDeclHeader(className: List[String]): Unit = {} def instanceClear(instName: InstanceIdentifier): Unit = {} def instanceSetCalculated(instName: InstanceIdentifier): Unit = {} - def instanceDeclaration(attrName: InstanceIdentifier, attrType: DataType, condSpec: ConditionalSpec) = attributeDeclaration(attrName, attrType, condSpec) - def instanceHeader(className: List[String], instName: InstanceIdentifier, dataType: DataType): Unit + def instanceDeclaration(attrName: InstanceIdentifier, attrType: DataType, isNullable: Boolean): Unit = attributeDeclaration(attrName, attrType, isNullable) + def instanceHeader(className: List[String], instName: InstanceIdentifier, dataType: DataType, isNullable: Boolean): Unit def instanceFooter: Unit - def instanceCheckCacheAndReturn(instName: InstanceIdentifier): Unit - def instanceReturn(instName: InstanceIdentifier): Unit - def instanceCalculate(instName: InstanceIdentifier, dataType: DataType, value: Ast.expr) + def instanceCheckCacheAndReturn(instName: InstanceIdentifier, dataType: DataType): Unit + def instanceReturn(instName: InstanceIdentifier, attrType: DataType): Unit + def instanceCalculate(instName: Identifier, dataType: DataType, value: Ast.expr) - def enumDeclaration(curClass: List[String], enumName: String, enumColl: Seq[(Long, String)]): Unit + def enumDeclaration(curClass: List[String], enumName: String, enumColl: Seq[(Long, EnumValueSpec)]): Unit /** * Outputs class' attributes sequence identifiers as some sort of an ordered sequence, diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompilerStatic.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompilerStatic.scala index 8a678dbcc..12e44167b 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompilerStatic.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompilerStatic.scala @@ -2,7 +2,6 @@ package io.kaitai.struct.languages.components import io.kaitai.struct._ import io.kaitai.struct.languages._ -import io.kaitai.struct.translators.{BaseTranslator, TypeProvider} trait LanguageCompilerStatic { def getCompiler(tp: ClassTypeProvider, config: RuntimeConfig): LanguageCompiler @@ -10,15 +9,21 @@ trait LanguageCompilerStatic { object LanguageCompilerStatic { val NAME_TO_CLASS: Map[String, LanguageCompilerStatic] = Map( + "construct" -> ConstructClassCompiler, "cpp_stl" -> CppCompiler, "csharp" -> CSharpCompiler, "graphviz" -> GraphvizClassCompiler, + "go" -> GoCompiler, + "html" -> HtmlClassCompiler, "java" -> JavaCompiler, "javascript" -> JavaScriptCompiler, + "lua" -> LuaCompiler, + "nim" -> NimClassCompiler, "perl" -> PerlCompiler, "php" -> PHPCompiler, "python" -> PythonCompiler, - "ruby" -> RubyCompiler + "ruby" -> RubyCompiler, + "rust" -> RustCompiler ) val CLASS_TO_NAME: Map[LanguageCompilerStatic, String] = NAME_TO_CLASS.map(_.swap) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/NoNeedForFullClassPath.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/NoNeedForFullClassPath.scala index 963f47a20..63be66119 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/NoNeedForFullClassPath.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/NoNeedForFullClassPath.scala @@ -1,7 +1,7 @@ package io.kaitai.struct.languages.components import io.kaitai.struct.datatype.DataType -import io.kaitai.struct.format.InstanceIdentifier +import io.kaitai.struct.format._ trait NoNeedForFullClassPath { def classHeader(name: List[String]): Unit = @@ -12,15 +12,15 @@ trait NoNeedForFullClassPath { classFooter(name.last) def classFooter(name: String): Unit - def classConstructorHeader(name: List[String], parentClassName: List[String], rootClassName: List[String]): Unit = - classConstructorHeader(name.last, parentClassName.last, rootClassName.last) - def classConstructorHeader(name: String, parentClassName: String, rootClassName: String): Unit + def classConstructorHeader(name: List[String], parentType: DataType, rootClassName: List[String], isHybrid: Boolean, params: List[ParamDefSpec]): Unit = + classConstructorHeader(name.last, parentType, rootClassName.last, isHybrid, params) + def classConstructorHeader(name: String, parentType: DataType, rootClassName: String, isHybrid: Boolean, params: List[ParamDefSpec]): Unit - def instanceHeader(className: List[String], instName: InstanceIdentifier, dataType: DataType): Unit = - instanceHeader(className.last, instName, dataType) - def instanceHeader(className: String, instName: InstanceIdentifier, dataType: DataType): Unit + def instanceHeader(className: List[String], instName: InstanceIdentifier, dataType: DataType, isNullable: Boolean): Unit = + instanceHeader(className.last, instName, dataType, isNullable) + def instanceHeader(className: String, instName: InstanceIdentifier, dataType: DataType, isNullable: Boolean): Unit - def enumDeclaration(curClass: List[String], enumName: String, enumColl: Seq[(Long, String)]): Unit = - enumDeclaration(curClass.last, enumName, enumColl) + def enumDeclaration(curClass: List[String], enumName: String, enumColl: Seq[(Long, EnumValueSpec)]): Unit = + enumDeclaration(curClass.last, enumName, enumColl.map((x) => (x._1, x._2.name))) def enumDeclaration(curClass: String, enumName: String, enumColl: Seq[(Long, String)]): Unit } diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/ObjectOrientedLanguage.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/ObjectOrientedLanguage.scala index 30dcf2da4..02d00c943 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/ObjectOrientedLanguage.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/ObjectOrientedLanguage.scala @@ -39,5 +39,23 @@ trait ObjectOrientedLanguage extends LanguageCompiler { */ def publicMemberName(id: Identifier): String + /** + * Renders identifier as a proper reference to a local temporary + * variable appropriately named to hold a temporary reference to + * this field. + * + * @param id identifier to render + * @return identifier as string + */ + def localTemporaryName(id: Identifier): String + + /** + * Renders identifier as a parameter (method argument) name. + * Default implementation just calls [[idToStr]]. + * @param id + * @return + */ + def paramName(id: Identifier): String = idToStr(id) + override def normalIO: String = privateMemberName(IoIdentifier) } diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/SingleOutputFile.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/SingleOutputFile.scala index cf154f286..11d4c77ae 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/SingleOutputFile.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/SingleOutputFile.scala @@ -1,14 +1,32 @@ package io.kaitai.struct.languages.components -import io.kaitai.struct.StringLanguageOutputWriter +import io.kaitai.struct.{ImportList, StringLanguageOutputWriter, Utils} import io.kaitai.struct.format.ClassSpec +import scala.collection.mutable.ListBuffer + /** * Common trait for languages that have one output file per ClassSpec. + * This file is considered to be composed of: + * + * * a header + * * imports list + * * output body */ trait SingleOutputFile extends LanguageCompiler { + val outHeader = new StringLanguageOutputWriter(indent) val out = new StringLanguageOutputWriter(indent) override def results(topClass: ClassSpec) = - Map(outFileName(topClass.nameAsStr) -> out.result) + Map(outFileName(topClass.nameAsStr) -> + (outHeader.result + outImports(topClass) + out.result) + ) + + val importList = new ImportList + + /** + * Generates imports clauses in target language format + * @return import + */ + def outImports(topClass: ClassSpec) = "" } diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/SwitchIfOps.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/SwitchIfOps.scala new file mode 100644 index 000000000..d490eb3a0 --- /dev/null +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/SwitchIfOps.scala @@ -0,0 +1,109 @@ +package io.kaitai.struct.languages.components + +import io.kaitai.struct.ClassTypeProvider +import io.kaitai.struct.datatype.DataType +import io.kaitai.struct.datatype.DataType.SwitchType +import io.kaitai.struct.exprlang.Ast +import io.kaitai.struct.format.Identifier +import io.kaitai.struct.translators.BaseTranslator + +/** + * Trait to be used when language needs two implementation of switching: + * a "true" one utilizing switch-like statement and an "if emulation" + * which utilizes series of if-then-else statements to emulate a switch. + * + * "True" switches are typically more efficient, but are limited to a + * subset of types. Examples of consumers of this pattern are C++, C#, Java. + */ +trait SwitchIfOps extends SwitchOps { + val translator: BaseTranslator + def typeProvider: ClassTypeProvider + + /** + * Determines if this particular implementation of switches would be ok with true + * built-in `switch` mechanism, or it will require `if`-based emulation. + * + * @param onType type we'll be switching over + * @return true if `if`-based emulation is required + */ + def switchRequiresIfs(onType: DataType): Boolean + + def switchIfStart(id: Identifier, on: Ast.expr, onType: DataType): Unit + def switchIfCaseFirstStart(condition: Ast.expr): Unit = switchIfCaseStart(condition) + def switchIfCaseStart(condition: Ast.expr): Unit + def switchIfCaseEnd(): Unit + def switchIfElseStart(): Unit + def switchIfElseEnd(): Unit = switchIfCaseEnd() + def switchIfEnd(): Unit + + /** + * Generate switch cases by calling case procedures. Suitable for a wide variety of + * target languages that something remotely resembling C-like `switch`-`case` statement. + * Thanks to customizable argument type for case procedures, can be used for switch type + * handling and a variety of other cases (i.e. switching between customizable endianness, + * etc). + * @param id attribute identifier + * @param on on expression to decide upon + * @param cases cases map: keys should be expressions, values are arbitrary typed objects + * that will be passed to case procedures + * @param normalCaseProc procedure that would handle "normal" (i.e. non-else case) + * @param elseCaseProc procedure that would handle "else" case + * @tparam T type of object to pass to procedures + */ + override def switchCases[T]( + id: Identifier, + on: Ast.expr, + cases: Map[Ast.expr, T], + normalCaseProc: (T) => Unit, + elseCaseProc: (T) => Unit + ): Unit = { + val onType = translator.detectType(on) + typeProvider._currentSwitchType = Some(onType) + val switchIfs = switchRequiresIfs(onType) + + if (switchIfs) { + switchCasesUsingIf(id, on, onType, cases, normalCaseProc, elseCaseProc) + } else { + switchCasesRender(id, on, cases, normalCaseProc, elseCaseProc) + } + } + + protected def switchCasesUsingIf[T]( + id: Identifier, + on: Ast.expr, + onType: DataType, + cases: Map[Ast.expr, T], + normalCaseProc: (T) => Unit, + elseCaseProc: (T) => Unit + ): Unit = { + switchIfStart(id, on, onType) + + // Pass 1: only normal case clauses + var first = true + + cases.foreach { case (condition, result) => + condition match { + case SwitchType.ELSE_CONST => + // skip for now + case _ => + if (first) { + switchIfCaseFirstStart(condition) + first = false + } else { + switchIfCaseStart(condition) + } + normalCaseProc(result) + switchIfCaseEnd() + } + } + + // Pass 2: else clause, if it is there + cases.get(SwitchType.ELSE_CONST).foreach { (result) => + switchIfElseStart() + elseCaseProc(result) + switchIfElseEnd() + } + + switchIfEnd() + } +} diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/SwitchOps.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/SwitchOps.scala new file mode 100644 index 000000000..36d80f88a --- /dev/null +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/SwitchOps.scala @@ -0,0 +1,90 @@ +package io.kaitai.struct.languages.components + +import io.kaitai.struct.datatype.DataType.SwitchType +import io.kaitai.struct.exprlang.Ast +import io.kaitai.struct.format.Identifier + +/** + * An interface for switching operations. + */ +trait SwitchOps { + def switchStart(id: Identifier, on: Ast.expr): Unit + def switchCaseFirstStart(condition: Ast.expr): Unit = switchCaseStart(condition) + def switchCaseStart(condition: Ast.expr): Unit + def switchCaseEnd(): Unit + def switchElseStart(): Unit + def switchElseEnd(): Unit = switchCaseEnd() + def switchEnd(): Unit + + /** + * Controls parsing of typeless (BytesType) alternative in switch case. If true, + * then target language does not support storing both bytes array and true object + * in the same variable, so we'll use workaround: bytes array will be read as + * _raw_ variable (which would be used anyway for all other cases as well). If + * false (which is default), we'll store *both* true objects and bytes array in + * the same variable. + */ + def switchBytesOnlyAsRaw = false + + /** + * Generate switch cases by calling case procedures. Suitable for a wide variety of + * target languages that something remotely resembling C-like `switch`-`case` statement. + * Thanks to customizable argument type for case procedures, can be used for switch type + * handling and a variety of other cases (i.e. switching between customizable endianness, + * etc). + * @param id attribute identifier + * @param on on expression to decide upon + * @param cases cases map: keys should be expressions, values are arbitrary typed objects + * that will be passed to case procedures + * @param normalCaseProc procedure that would handle "normal" (i.e. non-else case) + * @param elseCaseProc procedure that would handle "else" case + * @tparam T type of object to pass to procedures + */ + def switchCases[T]( + id: Identifier, + on: Ast.expr, + cases: Map[Ast.expr, T], + normalCaseProc: (T) => Unit, + elseCaseProc: (T) => Unit + ): Unit = { + switchCasesRender(id, on, cases, normalCaseProc, elseCaseProc) + } + + protected def switchCasesRender[T]( + id: Identifier, + on: Ast.expr, + cases: Map[Ast.expr, T], + normalCaseProc: (T) => Unit, + elseCaseProc: (T) => Unit + ): Unit = { + switchStart(id, on) + + // Pass 1: only normal case clauses + var first = true + + cases.foreach { case (condition, result) => + condition match { + case SwitchType.ELSE_CONST => + // skip for now + case _ => + if (first) { + switchCaseFirstStart(condition) + first = false + } else { + switchCaseStart(condition) + } + normalCaseProc(result) + switchCaseEnd() + } + } + + // Pass 2: else clause, if it is there + cases.get(SwitchType.ELSE_CONST).foreach { (result) => + switchElseStart() + elseCaseProc(result) + switchElseEnd() + } + + switchEnd() + } +} diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/UniversalFooter.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/UniversalFooter.scala index cdde62343..e356678c9 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/UniversalFooter.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/UniversalFooter.scala @@ -15,8 +15,9 @@ trait UniversalFooter extends LanguageCompiler { def classFooter(name: String): Unit = universalFooter def classConstructorFooter: Unit = universalFooter - override def funcWriteFooter(curClass: ClassSpec): Unit = universalFooter - override def funcCheckFooter(curClass: ClassSpec): Unit = universalFooter + override def readFooter: Unit = universalFooter + override def writeFooter: Unit = universalFooter + override def checkFooter: Unit = universalFooter def condRepeatExprFooter = universalFooter def condRepeatEosFooter: Unit = universalFooter def condIfFooter(expr: expr): Unit = universalFooter diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/ValidateOps.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/ValidateOps.scala new file mode 100644 index 000000000..6fa04e184 --- /dev/null +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/ValidateOps.scala @@ -0,0 +1,34 @@ +package io.kaitai.struct.languages.components + +import io.kaitai.struct.datatype.{DataType, KSError, ValidationNotEqualError} +import io.kaitai.struct.exprlang.Ast +import io.kaitai.struct.format.{AttrSpec, Identifier, IoIdentifier, ValidationEq, ValidationSpec, YAMLParseException} + +/** + * Common interface for validation operations. + */ +trait ValidateOps extends ExceptionNames { + def attrValidate(attrId: Identifier, attr: AttrSpec, valid: ValidationSpec): Unit = { + valid match { + case ValidationEq(expected) => + attrValidateExpr( + attrId, + attr.dataType, + Ast.expr.Compare( + Ast.expr.Name(attrId.toAstIdentifier), + Ast.cmpop.Eq, + expected + ), + ksErrorName(ValidationNotEqualError(attr.dataType)), + List( + expected, + Ast.expr.Name(attrId.toAstIdentifier), + Ast.expr.Name(IoIdentifier.toAstIdentifier), + Ast.expr.Str(attr.path.mkString("/", "/", "")) + ) + ) + } + } + + def attrValidateExpr(attrId: Identifier, attrType: DataType, checkExpr: Ast.expr, errName: String, errArgs: List[Ast.expr]): Unit = {} +} diff --git a/shared/src/main/scala/io/kaitai/struct/precompile/CalculateSeqSizes.scala b/shared/src/main/scala/io/kaitai/struct/precompile/CalculateSeqSizes.scala new file mode 100644 index 000000000..84f8c0260 --- /dev/null +++ b/shared/src/main/scala/io/kaitai/struct/precompile/CalculateSeqSizes.scala @@ -0,0 +1,120 @@ +package io.kaitai.struct.precompile + +import io.kaitai.struct.Log +import io.kaitai.struct.datatype.DataType +import io.kaitai.struct.datatype.DataType._ +import io.kaitai.struct.exprlang.Ast +import io.kaitai.struct.format._ + +class CalculateSeqSizes(specs: ClassSpecs) { + def run(): Unit = specs.forEachRec(CalculateSeqSizes.getSeqSize) +} + +object CalculateSeqSizes { + def sizeMultiply(sizeElement: Sized, repeat: RepeatSpec) = { + sizeElement match { + case FixedSized(elementSize) => + repeat match { + case NoRepeat => + sizeElement + case RepeatExpr(expr) => + expr.evaluateIntConst match { + case Some(count) => FixedSized(elementSize * count.toInt) + case None => DynamicSized + } + case _: RepeatUntil | RepeatEos => + DynamicSized + } + case _ => sizeElement + } + } + + def getSeqSize(curClass: ClassSpec): Sized = { + curClass.seqSize match { + case DynamicSized | _: FixedSized => + // do nothing, it's already calculated + case StartedCalculationSized => + // recursive size dependency encountered => we won't be able to determine + // let's break the infinite loop + curClass.seqSize = DynamicSized + case NotCalculatedSized => + // launch the calculation + curClass.seqSize = StartedCalculationSized + val seqSize = forEachSeqAttr(curClass, (attr, seqPos, sizeElement, sizeContainer) => {}) + curClass.seqSize = seqSize match { + case Some(size) => FixedSized(size) + case None => DynamicSized + } + } + + Log.seqSizes.info(() => s"sizeof(${curClass.nameAsStr}) = ${curClass.seqSize}") + curClass.seqSize + } + + /** + * Traverses type's sequence of attributes, calling operation for every attribute. + * Operation is called with arguments (attr, seqPos, sizeElement, sizeContainer) + * @param curClass type specification to traverse + * @param op operation to apply to every sequence attribute + * @return total size of sequence, if possible (i.e. it's fixed size) + */ + def forEachSeqAttr(curClass: ClassSpec, op: (AttrSpec, Option[Int], Sized, Sized) => Unit): Option[Int] = { + var seqPos: Option[Int] = Some(0) + curClass.seq.foreach { attr => + val sizeElement = dataTypeBitsSize(attr.dataType) + val sizeContainer = sizeMultiply(sizeElement, attr.cond.repeat) + + op(attr, seqPos, sizeElement, sizeContainer) + + seqPos = (seqPos, sizeContainer) match { + case (Some(pos), FixedSized(siz)) => Some(pos + siz) + case _ => None + } + } + seqPos + } + + /** + * Determines how many bits occupies given data type. + * + * @param dataType data type to analyze + * @return number of bits or None, if it's impossible to determine a priori + */ + def dataTypeBitsSize(dataType: DataType): Sized = { + dataType match { + case BitsType1 => FixedSized(1) + case BitsType(width) => FixedSized(width) + case EnumType(_, basedOn) => dataTypeBitsSize(basedOn) + case ut: UserTypeInstream => getSeqSize(ut.classSpec.get) + case _ => + dataTypeByteSize(dataType) match { + case FixedSized(x) => FixedSized(x * 8) + case otherSize => otherSize + } + } + } + + /** + * Determines how many bytes occupies a given data type. + * + * @param dataType data type to analyze + * @return number of bytes or None, if it's impossible to determine a priori + */ + def dataTypeByteSize(dataType: DataType): Sized = { + dataType match { + case _: Int1Type => FixedSized(1) + case IntMultiType(_, width, _) => FixedSized(width.width) + case FixedBytesType(contents, _) => FixedSized(contents.length) + case FloatMultiType(width, _) => FixedSized(width.width) + case _: BytesEosType => DynamicSized + case blt: BytesLimitType => blt.size.evaluateIntConst match { + case Some(x) => FixedSized(x.toInt) + case None => DynamicSized + } + case _: BytesTerminatedType => DynamicSized + case StrFromBytesType(basedOn, _) => dataTypeByteSize(basedOn) + case utb: UserTypeFromBytes => dataTypeByteSize(utb.bytes) + case st: SwitchType => DynamicSized // FIXME: it's really possible get size if st.hasSize + } + } +} diff --git a/shared/src/main/scala/io/kaitai/struct/precompile/Exceptions.scala b/shared/src/main/scala/io/kaitai/struct/precompile/Exceptions.scala index 586dc42b7..7425672bd 100644 --- a/shared/src/main/scala/io/kaitai/struct/precompile/Exceptions.scala +++ b/shared/src/main/scala/io/kaitai/struct/precompile/Exceptions.scala @@ -1,5 +1,6 @@ package io.kaitai.struct.precompile +import io.kaitai.struct.datatype.DataType import io.kaitai.struct.format.ClassSpec /** @@ -9,7 +10,7 @@ import io.kaitai.struct.format.ClassSpec * @param path YAML path components in file * @param file file to report as erroneous, None means "main compilation unit" */ -class ErrorInInput(err: Throwable, path: List[String] = List(), file: Option[String] = None) +case class ErrorInInput(err: Throwable, val path: List[String] = List(), val file: Option[String] = None) extends RuntimeException(ErrorInInput.message(err, path, file), err) object ErrorInInput { @@ -18,7 +19,8 @@ object ErrorInInput { case Some(x) => x.replace('\\', '/') case None => "(main)" } - s"$fileStr: /${path.mkString("/")}: ${err.getMessage}" + val msg = Option(err.getMessage).getOrElse(err.toString) + s"$fileStr: /${path.mkString("/")}: $msg" } } @@ -37,3 +39,12 @@ class FieldNotFoundError(val name: String, val curClass: ClassSpec) extends NotFoundError(s"unable to access '$name' in ${curClass.nameAsStr} context") class EnumNotFoundError(val name: String, val curClass: ClassSpec) extends NotFoundError(s"unable to find enum '$name', searching from ${curClass.nameAsStr}") +class MethodNotFoundError(val name: String, val dataType: DataType) + extends NotFoundError(s"don't know how to call method '$name' of object type '$dataType'") + +/** + * Internal compiler logic error: should never happen, but at least we want to + * handle it gracefully if it's happening. + * @param msg message for the user + */ +case class InternalCompilerError(msg: String) extends RuntimeException(msg) diff --git a/shared/src/main/scala/io/kaitai/struct/precompile/LoadImports.scala b/shared/src/main/scala/io/kaitai/struct/precompile/LoadImports.scala index 1a7649da0..bd9684704 100644 --- a/shared/src/main/scala/io/kaitai/struct/precompile/LoadImports.scala +++ b/shared/src/main/scala/io/kaitai/struct/precompile/LoadImports.scala @@ -13,6 +13,8 @@ import scala.concurrent.Future * @param specs collection of [[ClassSpec]] entries to work on */ class LoadImports(specs: ClassSpecs) { + import LoadImports._ + /** * Recursively loads and processes all .ksy files referenced in * `meta/import` section of given class spec and all nested classes. @@ -22,36 +24,52 @@ class LoadImports(specs: ClassSpecs) { * * @param curClass class spec to start recursive import from */ - def processClass(curClass: ClassSpec): Future[List[ClassSpec]] = { - Log.importOps.info(() => s".. LoadImports: processing class ${curClass.nameAsStr}") - - val thisMetaFuture: Future[List[ClassSpec]] = curClass.meta match { - case Some(meta) => - Future.sequence(meta.imports.zipWithIndex.map { case (name, idx) => - loadImport(name, meta.path ++ List("imports", idx.toString), Some(curClass.nameAsStr)) - }).map((x) => x.flatten) - case None => - Future { List() } - } + def processClass(curClass: ClassSpec, workDir: ImportPath): Future[List[ClassSpec]] = { + Log.importOps.info(() => s".. LoadImports: processing class ${curClass.nameAsStr} (workDir = $workDir)") + + val thisMetaFuture: Future[List[ClassSpec]] = + Future.sequence(curClass.meta.imports.zipWithIndex.map { case (name, idx) => + loadImport( + name, + curClass.meta.path ++ List("imports", idx.toString), + Some(curClass.nameAsStr), + workDir + ) + }).map((x) => x.flatten) val nestedFuture: Future[Iterable[ClassSpec]] = Future.sequence(curClass.types.map({ - case (_, nestedClass) => processClass(nestedClass) + case (_, nestedClass) => processClass(nestedClass, workDir) })).map((listOfLists) => listOfLists.flatten) Future.sequence(List(thisMetaFuture, nestedFuture)).map((x) => x.flatten) } - private def loadImport(name: String, path: List[String], inFile: Option[String]): Future[List[ClassSpec]] = { - val futureSpec = if (name.startsWith("/")) { - specs.importAbsolute(name.substring(1), path, inFile) - } else { - specs.importRelative(name, path, inFile) + private def loadImport(name: String, path: List[String], inFile: Option[String], workDir: ImportPath): Future[List[ClassSpec]] = { + Log.importOps.info(() => s".. LoadImports: loadImport($name, workDir = $workDir)") + + val impPath = ImportPath.fromString(name) + val fullPath = ImportPath.add(workDir, impPath) + + val futureSpec = fullPath match { + case RelativeImportPath(p) => + specs.importRelative(p.mkString("/"), path, inFile) + case AbsoluteImportPath(p) => + specs.importAbsolute(p.mkString("/"), path, inFile) } futureSpec.flatMap { case optSpec => + Log.importOps.info(() => { + val specNameAsStr = optSpec.map(_.nameAsStr).getOrElse("") + s".. LoadImports: loadImport($name, workDir = $workDir), got spec=$specNameAsStr" + }) optSpec match { case Some(spec) => val specName = spec.name.head + // Check if spec name does not match file name. If it doesn't match, + // it is probably already a serious error. + if (name != specName) + Log.importOps.warn(() => s"... expected to have type name $name, but got $specName") + // Check if we've already had this spec in our ClassSpecs. If we do, // don't do anything: we've already processed it and reprocessing it // might lead to infinite recursion. @@ -60,8 +78,9 @@ class LoadImports(specs: ClassSpecs) { // import* methods due to caching, but we won't rely on it here. if (!specs.contains(specName)) { specs(specName) = spec - processClass(spec) + processClass(spec, ImportPath.updateWorkDir(workDir, impPath)) } else { + Log.importOps.warn(() => s"... we have that already, ignoring") Future { List() } } case None => @@ -70,3 +89,38 @@ class LoadImports(specs: ClassSpecs) { } } } + +object LoadImports { + sealed trait ImportPath { + def baseDir: ImportPath + } + case class RelativeImportPath(path: List[String]) extends ImportPath { + override def baseDir: ImportPath = RelativeImportPath(path.init) + } + case class AbsoluteImportPath(path: List[String]) extends ImportPath { + override def baseDir: ImportPath = AbsoluteImportPath(path.init) + } + val BasePath = RelativeImportPath(List()) + + object ImportPath { + def fromString(s: String): ImportPath = if (s.startsWith("/")) { + AbsoluteImportPath(s.substring(1).split("/", -1).toList) + } else { + RelativeImportPath(s.split("/", -1).toList) + } + + def add(curWorkDir: ImportPath, newPath: ImportPath): ImportPath = { + (curWorkDir, newPath) match { + case (_, AbsoluteImportPath(newPathAbs)) => + AbsoluteImportPath(newPathAbs) + case (RelativeImportPath(curDir), RelativeImportPath(newPathRel)) => + RelativeImportPath(curDir ++ newPathRel) + case (AbsoluteImportPath(curDir), RelativeImportPath(newPathRel)) => + AbsoluteImportPath(curDir ++ newPathRel) + } + } + + def updateWorkDir(curWorkDir: ImportPath, newPath: ImportPath): ImportPath = + add(curWorkDir, newPath).baseDir + } +} diff --git a/shared/src/main/scala/io/kaitai/struct/precompile/ParentTypes.scala b/shared/src/main/scala/io/kaitai/struct/precompile/ParentTypes.scala index 0a0beebf5..63f242196 100644 --- a/shared/src/main/scala/io/kaitai/struct/precompile/ParentTypes.scala +++ b/shared/src/main/scala/io/kaitai/struct/precompile/ParentTypes.scala @@ -3,11 +3,14 @@ package io.kaitai.struct.precompile import io.kaitai.struct.{ClassTypeProvider, Log} import io.kaitai.struct.datatype.DataType import io.kaitai.struct.datatype.DataType.{ArrayType, SwitchType, UserType} -import io.kaitai.struct.exprlang.Ast import io.kaitai.struct.format._ import io.kaitai.struct.translators.TypeDetector -object ParentTypes { +class ParentTypes(classSpecs: ClassSpecs) { + def run(): Unit = { + classSpecs.foreach { case (_, curClass) => markup(curClass) } + } + def markup(curClass: ClassSpec): Unit = { Log.typeProcParent.info(() => s"markupParentTypes(${curClass.nameAsStr})") @@ -40,7 +43,7 @@ object ParentTypes { Log.typeProcParent.info(() => s"..... no parent type added") None case Some(parent) => - val provider = new ClassTypeProvider(curClass) + val provider = new ClassTypeProvider(classSpecs, curClass) val detector = new TypeDetector(provider) val parentType = detector.detectType(parent) Log.typeProcParent.info(() => s"..... enforced parent type = $parentType") diff --git a/shared/src/main/scala/io/kaitai/struct/precompile/ResolveTypes.scala b/shared/src/main/scala/io/kaitai/struct/precompile/ResolveTypes.scala index 37ba4d487..d221c1890 100644 --- a/shared/src/main/scala/io/kaitai/struct/precompile/ResolveTypes.scala +++ b/shared/src/main/scala/io/kaitai/struct/precompile/ResolveTypes.scala @@ -10,11 +10,7 @@ import io.kaitai.struct.format._ * converts names into ClassSpec / EnumSpec references. */ class ResolveTypes(specs: ClassSpecs, opaqueTypes: Boolean) { - def run(): Unit = - specs.foreach { case (_, spec) => - // FIXME: grab exception and rethrow more localized one, with a specName? - resolveUserTypes(spec) - } + def run(): Unit = specs.forEachRec(resolveUserTypes) /** * Resolves user types and enum types recursively starting from a certain @@ -22,23 +18,21 @@ class ResolveTypes(specs: ClassSpecs, opaqueTypes: Boolean) { * @param curClass class to start from, might be top-level class */ def resolveUserTypes(curClass: ClassSpec): Unit = { - curClass.seq.foreach((attr) => resolveUserTypeForAttr(curClass, attr)) + curClass.seq.foreach((attr) => resolveUserTypeForMember(curClass, attr)) curClass.instances.foreach { case (_, instSpec) => instSpec match { case pis: ParseInstanceSpec => - resolveUserTypeForAttr(curClass, pis) + resolveUserTypeForMember(curClass, pis) case _: ValueInstanceSpec => // ignore all other types of instances } } - curClass.types.foreach { case (_, nestedClass) => - resolveUserTypes(nestedClass) - } + curClass.params.foreach((paramDef) => resolveUserTypeForMember(curClass, paramDef)) } - def resolveUserTypeForAttr(curClass: ClassSpec, attr: AttrLikeSpec): Unit = + def resolveUserTypeForMember(curClass: ClassSpec, attr: MemberSpec): Unit = resolveUserType(curClass, attr.dataType, attr.path) def resolveUserType(curClass: ClassSpec, dataType: DataType, path: List[String]): Unit = { @@ -51,8 +45,8 @@ class ResolveTypes(specs: ClassSpecs, opaqueTypes: Boolean) { val err = new EnumNotFoundError(et.name.mkString("::"), curClass) throw new YAMLParseException(err.getMessage, path) } - case SwitchType(_, cases) => - cases.foreach { case (caseName, ut) => + case st: SwitchType => + st.cases.foreach { case (caseName, ut) => resolveUserType(curClass, ut, path ++ List("type", "cases", caseName.toString)) } case _ => @@ -113,7 +107,15 @@ class ResolveTypes(specs: ClassSpecs, opaqueTypes: Boolean) { } else { // Check if top-level specs has this name // If there's None => no luck at all - specs.get(firstName) + val resolvedTop = specs.get(firstName) + resolvedTop match { + case None => None + case Some(classSpec) => if (restNames.isEmpty) { + resolvedTop + } else { + resolveUserType(classSpec, restNames, path) + } + } } } } diff --git a/shared/src/main/scala/io/kaitai/struct/precompile/SpecsValueTypeDerive.scala b/shared/src/main/scala/io/kaitai/struct/precompile/SpecsValueTypeDerive.scala index 5d10845a5..74686d1a1 100644 --- a/shared/src/main/scala/io/kaitai/struct/precompile/SpecsValueTypeDerive.scala +++ b/shared/src/main/scala/io/kaitai/struct/precompile/SpecsValueTypeDerive.scala @@ -12,7 +12,7 @@ class SpecsValueTypeDerive(specs: ClassSpecs) { Log.typeProcValue.info(() => s"### SpecsValueTypeDerive: iteration #$iterNum") specs.foreach { case (specName, spec) => Log.typeProcValue.info(() => s"#### $specName") - val thisChanged = new ValueTypesDeriver(spec).run() + val thisChanged = new ValueTypesDeriver(specs, spec).run() Log.typeProcValue.info(() => ".... => " + (if (thisChanged) "changed" else "no changes")) hasChanged |= thisChanged } diff --git a/shared/src/main/scala/io/kaitai/struct/precompile/TypeValidator.scala b/shared/src/main/scala/io/kaitai/struct/precompile/TypeValidator.scala index d6f37d613..54de23429 100644 --- a/shared/src/main/scala/io/kaitai/struct/precompile/TypeValidator.scala +++ b/shared/src/main/scala/io/kaitai/struct/precompile/TypeValidator.scala @@ -1,36 +1,44 @@ package io.kaitai.struct.precompile -import io.kaitai.struct.ClassTypeProvider +import io.kaitai.struct.{ClassTypeProvider, Log} import io.kaitai.struct.datatype.DataType import io.kaitai.struct.datatype.DataType._ import io.kaitai.struct.exprlang.Ast import io.kaitai.struct.format._ -import io.kaitai.struct.translators.TypeDetector +import io.kaitai.struct.translators.{ExpressionValidator, TypeDetector} import scala.reflect.ClassTag /** * Validates all expressions used inside the given ClassSpec to use expected types. + * Also ensures that all expressions usage of types (in typecasting operator, + * enums, sizeof operator, etc) matches loaded classes layout & names. + * @param specs bundle of class specifications (used only to find external references) * @param topClass class to start check with */ -class TypeValidator(topClass: ClassSpec) { - val provider = new ClassTypeProvider(topClass) - val detector = new TypeDetector(provider) +class TypeValidator(specs: ClassSpecs, topClass: ClassSpec) { + val provider = new ClassTypeProvider(specs, topClass) + val detector = new ExpressionValidator(provider) /** * Starts the check from top-level class. */ - def run(): Unit = - validateClass(topClass) + def run(): Unit = specs.forEachTopLevel { (specName, curClass) => + Log.typeValid.info(() => s"validating top level class '$specName'") + provider.topClass = curClass + curClass.forEachRec(validateClass) + } /** * Performs validation of a single ClassSpec: would validate * sequence attributes (`seq`), instances (`instances`) and all * nested subtypes (`types`) recursively. `doc` and `enums` are * not checked, as they contain no expressions. + * * @param curClass class to check */ def validateClass(curClass: ClassSpec): Unit = { + Log.typeValid.info(() => s"validateClass(${curClass.nameAsStr})") provider.nowClass = curClass curClass.seq.foreach(validateAttr) @@ -38,15 +46,11 @@ class TypeValidator(topClass: ClassSpec) { curClass.instances.foreach { case (_, inst) => inst match { case pis: ParseInstanceSpec => - validateAttr(pis) + validateParseInstance(pis) case vis: ValueInstanceSpec => - // TODO + validateValueInstance(vis) } } - - curClass.types.foreach { case (_, nestedClass) => - validateClass(nestedClass) - } } /** @@ -55,6 +59,8 @@ class TypeValidator(topClass: ClassSpec) { * @param attr attribute to check */ def validateAttr(attr: AttrLikeSpec) { + Log.typeValid.info(() => s"validateAttr(${attr.id.humanReadable})") + val path = attr.path attr.cond.ifExpr.foreach((ifExpr) => @@ -74,13 +80,49 @@ class TypeValidator(topClass: ClassSpec) { validateDataType(attr.dataType, path) } + def validateParseInstance(pis: ParseInstanceSpec): Unit = { + validateAttr(pis) + + Log.typeValid.info(() => s"validateParseInstance(${pis.id.humanReadable})") + + pis.io match { + case Some(io) => checkAssertObject(io, KaitaiStreamType, "IO stream", pis.path, "io") + case None => // all good + } + + pis.pos match { + case Some(pos) => checkAssert[IntType](pos, "integer", pis.path, "pos") + case None => // all good + } + } + + def validateValueInstance(vis: ValueInstanceSpec): Unit = { + try { + detector.validate(vis.value) + } catch { + case err: ExpressionError => + throw new ErrorInInput(err, vis.path ++ List("value")) + } + } + /** * Validates single non-composite data type, checking all expressions * inside data type definition. + * * @param dataType data type to check * @param path original .ksy path to make error messages more meaningful */ def validateDataType(dataType: DataType, path: List[String]) { + // validate args vs params + dataType match { + case ut: UserType => + // we only validate non-opaque types, opaque are unverifiable by definition + if (!ut.isOpaque) + validateArgsVsParams(ut.args, ut.classSpec.get.params, path ++ List("type")) + case _ => + // no args or params in non-user types + } + dataType match { case blt: BytesLimitType => checkAssert[IntType](blt.size, "integer", path, "size") @@ -97,20 +139,49 @@ class TypeValidator(topClass: ClassSpec) { def validateSwitchType(st: SwitchType, path: List[String]) { val onType = detector.detectType(st.on) + detector.validate(st.on) st.cases.foreach { case (caseExpr, caseType) => val casePath = path ++ List("type", "cases", caseExpr.toString) if (caseExpr != SwitchType.ELSE_CONST) { try { TypeDetector.assertCompareTypes(onType, detector.detectType(caseExpr), Ast.cmpop.Eq) + detector.validate(caseExpr) } catch { case tme: TypeMismatchError => throw new YAMLParseException(tme.getMessage, casePath) + case err: Throwable => + throw new ErrorInInput(err, casePath) } } validateDataType(caseType, casePath) } } + /** + * Validates that arguments given for a certain type match list of parameters + * declared for that type. + * @param args arguments given in invocation + * @param params parameters declared in a user type + * @param path path where invocation happens + * @return + */ + def validateArgsVsParams(args: Seq[Ast.expr], params: List[ParamDefSpec], path: List[String]): Unit = { + if (args.size != params.size) + throw YAMLParseException.invalidParamCount(params.size, args.size, path) + + args.indices.foreach { (i) => + val arg = args(i) + val param = params(i) + val tArg = detector.detectType(arg) + detector.validate(arg) + val tParam = param.dataType + + if (!TypeDetector.canAssign(tArg, tParam)) { + throw YAMLParseException.paramMismatch(i, tArg, param.id.humanReadable, tParam, path) + } + } + } + /** * Checks that expression's type conforms to a given datatype, otherwise * throw a human-readable exception, with some pointers that would help @@ -134,11 +205,64 @@ class TypeValidator(topClass: ClassSpec) { try { detector.detectType(expr) match { case _: T => // good + case st: SwitchType => + st.combinedType match { + case _: T => // good + case actual => + throw YAMLParseException.exprType(expectStr, actual, path ++ List(pathKey)) + } case actual => throw YAMLParseException.exprType(expectStr, actual, path ++ List(pathKey)) } + detector.validate(expr) + } catch { + case err: InvalidIdentifier => + throw new ErrorInInput(err, path ++ List(pathKey)) + case err: ExpressionError => + throw new ErrorInInput(err, path ++ List(pathKey)) + } + } + + /** + * Checks that expression's type conforms to a given datatype, otherwise + * throw a human-readable exception, with some pointers that would help + * finding the expression in source .ksy. + * + * This version works with case objects. + * + * @param expr expression to check + * @param expectStr string to include + * @param path path to expression base + * @param pathKey key that contains expression in given path + */ + def checkAssertObject( + expr: Ast.expr, + expected: Object, + expectStr: String, + path: List[String], + pathKey: String + ): Unit = { + try { + val detected = detector.detectType(expr) + if (detected == expected) { + // good + } else { + detected match { + case st: SwitchType => + val combinedType = st.combinedType + if (combinedType == expected) { + // good + } else { + throw YAMLParseException.exprType(expectStr, combinedType, path ++ List(pathKey)) + } + case actual => throw YAMLParseException.exprType(expectStr, actual, path ++ List(pathKey)) + } + } + detector.validate(expr) } catch { + case err: InvalidIdentifier => + throw new ErrorInInput(err, path ++ List(pathKey)) case err: ExpressionError => - throw new YAMLParseException(err.getMessage, path ++ List(pathKey)) + throw new ErrorInInput(err, path ++ List(pathKey)) } } } diff --git a/shared/src/main/scala/io/kaitai/struct/precompile/ValueTypesDeriver.scala b/shared/src/main/scala/io/kaitai/struct/precompile/ValueTypesDeriver.scala index 6eaef26fd..eb1937025 100644 --- a/shared/src/main/scala/io/kaitai/struct/precompile/ValueTypesDeriver.scala +++ b/shared/src/main/scala/io/kaitai/struct/precompile/ValueTypesDeriver.scala @@ -1,11 +1,11 @@ package io.kaitai.struct.precompile -import io.kaitai.struct.format.{ClassSpec, ValueInstanceSpec, YAMLParseException} +import io.kaitai.struct.format.{ClassSpec, ClassSpecs, ValueInstanceSpec, YAMLParseException} import io.kaitai.struct.translators.TypeDetector import io.kaitai.struct.{ClassTypeProvider, Log} -class ValueTypesDeriver(topClass: ClassSpec) { - val provider = new ClassTypeProvider(topClass) +class ValueTypesDeriver(specs: ClassSpecs, topClass: ClassSpec) { + val provider = new ClassTypeProvider(specs, topClass) val detector = new TypeDetector(provider) def run(): Boolean = @@ -24,7 +24,7 @@ class ValueTypesDeriver(topClass: ClassSpec) { vi.dataType match { case None => try { - val viType = detector.detectType(vi.value) + val viType = detector.detectType(vi.value).asNonOwning vi.dataType = Some(viType) Log.typeProcValue.info(() => s"${instName.name} derived type: $viType") hasChanged = true diff --git a/shared/src/main/scala/io/kaitai/struct/translators/AbstractTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/AbstractTranslator.scala new file mode 100644 index 000000000..a96140ea7 --- /dev/null +++ b/shared/src/main/scala/io/kaitai/struct/translators/AbstractTranslator.scala @@ -0,0 +1,20 @@ +package io.kaitai.struct.translators + +import io.kaitai.struct.exprlang.Ast + +/** + * Translators are per-target language classes which implement translation + * of Kaitai Struct expression language into expression in target language. + * + * This simplest, most abstract form of translator provides only a single + * `translate` method, which takes KS expression and returns string in + * target language. + */ +trait AbstractTranslator { + /** + * Translates KS expression into an expression in some target language. + * @param v KS expression to translate + * @return expression in target language as string + */ + def translate(v: Ast.expr): String +} diff --git a/shared/src/main/scala/io/kaitai/struct/translators/BaseTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/BaseTranslator.scala index 7546825e8..bf93d1153 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/BaseTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/BaseTranslator.scala @@ -3,9 +3,49 @@ package io.kaitai.struct.translators import io.kaitai.struct.datatype.DataType import io.kaitai.struct.datatype.DataType._ import io.kaitai.struct.exprlang.Ast +import io.kaitai.struct.format.{ClassSpec, Identifier} import io.kaitai.struct.precompile.TypeMismatchError -abstract class BaseTranslator(val provider: TypeProvider) extends TypeDetector(provider) { +/** + * BaseTranslator is a common semi-abstract implementation of a translator + * API (i.e. [[AbstractTranslator]]), which fits target languages that + * follow "every KS expression is translatable into expression" paradigm. + * Main [[AbstractTranslator.translate]] method is implemented as a huge + * case matching, which usually just calls relevant abstract methods for + * every particular piece of KS expression, i.e. literals, operations, + * method calls, etc. + * + * Given that there are many of these abstract methods, to make it more + * maintainable, they are grouped into several abstract traits: + * [[CommonLiterals]], [[CommonOps]], [[CommonMethods]], + * [[CommonArraysAndCast]]. + * + * This translator implementation also handles user-defined types and + * fields properly - it uses given [[TypeProvider]] to resolve these. + * + * @param provider TypeProvider that will answer queries on user types + */ +abstract class BaseTranslator(val provider: TypeProvider) + extends TypeDetector(provider) + with AbstractTranslator + with CommonLiterals + with CommonOps + with CommonArraysAndCast[String] + with CommonMethods[String] + with ByteArraysAsTrueArrays[String] { + + /** + * Translates KS expression into an expression in some target language. + * Note that this implementation may throw errors subclassed off the + * [[io.kaitai.struct.precompile.ExpressionError]] when encountering + * some sort of logical error in expression (i.e. invalid usage of + * operator, type mismatch, etc). Typically, one's supposed to catch + * and rethrow it, wrapped in [[io.kaitai.struct.precompile.ErrorInInput]] + * to assist error reporting in KSC. + * + * @param v KS expression to translate + * @return expression in target language as string + */ def translate(v: Ast.expr): String = { v match { case Ast.expr.IntNum(n) => @@ -16,16 +56,25 @@ abstract class BaseTranslator(val provider: TypeProvider) extends TypeDetector(p doStringLiteral(s) case Ast.expr.Bool(n) => doBoolLiteral(n) - case Ast.expr.EnumById(enumType, id) => - val enumSpec = provider.resolveEnum(enumType.name) + case Ast.expr.EnumById(enumType, id, inType) => + val enumSpec = provider.resolveEnum(inType, enumType.name) doEnumById(enumSpec.name, translate(id)) - case Ast.expr.EnumByLabel(enumType, label) => - val enumSpec = provider.resolveEnum(enumType.name) + case Ast.expr.EnumByLabel(enumType, label, inType) => + val enumSpec = provider.resolveEnum(inType, enumType.name) doEnumByLabel(enumSpec.name, label.name) case Ast.expr.Name(name: Ast.identifier) => - doLocalName(name.name) - case Ast.expr.UnaryOp(op: Ast.unaryop, v: Ast.expr) => - s"${unaryOp(op)}${translate(v)}" + if (name.name == Identifier.SIZEOF) { + byteSizeOfClassSpec(provider.nowClass) + } else { + doLocalName(name.name) + } + case Ast.expr.UnaryOp(op: Ast.unaryop, inner: Ast.expr) => + unaryOp(op) + (inner match { + case Ast.expr.IntNum(_) | Ast.expr.FloatNum(_) => + translate(inner) + case _ => + s"(${translate(inner)})" + }) case Ast.expr.Compare(left: Ast.expr, op: Ast.cmpop, right: Ast.expr) => (detectType(left), detectType(right)) match { case (_: NumericType, _: NumericType) => @@ -42,9 +91,11 @@ abstract class BaseTranslator(val provider: TypeProvider) extends TypeDetector(p doStrCompareOp(left, op, right) case (_: BytesType, _: BytesType) => doBytesCompareOp(left, op, right) - case (EnumType(ltype, _), EnumType(rtype, _)) => - if (ltype != rtype) { - throw new TypeMismatchError(s"can't compare enums type $ltype and $rtype") + case (et1: EnumType, et2: EnumType) => + val et1Spec = et1.enumSpec.get + val et2Spec = et2.enumSpec.get + if (et1Spec != et2Spec) { + throw new TypeMismatchError(s"can't compare enums type ${et1Spec.nameAsStr} and ${et2Spec.nameAsStr}") } else { doEnumCompareOp(left, op, right) } @@ -67,264 +118,82 @@ abstract class BaseTranslator(val provider: TypeProvider) extends TypeDetector(p case Ast.expr.Subscript(container: Ast.expr, idx: Ast.expr) => detectType(idx) match { case _: IntType => - doSubscript(container, idx) + detectType(container) match { + case _: ArrayType => + arraySubscript(container, idx) + case _: BytesType => + bytesSubscript(container, idx) + case containerType => + throw new TypeMismatchError(s"can't index $containerType as array") + } case idxType => throw new TypeMismatchError(s"can't use $idx as array index (need int, got $idxType)") } - case Ast.expr.Attribute(value: Ast.expr, attr: Ast.identifier) => - val valType = detectType(value) - valType match { - case _: UserType => - userTypeField(value, attr.name) - case _: StrType => - attr.name match { - case "length" => strLength(value) - case "reverse" => strReverse(value) - case "to_i" => strToInt(value, Ast.expr.IntNum(10)) - } - case _: IntType => - attr.name match { - case "to_s" => intToStr(value, Ast.expr.IntNum(10)) - } - case ArrayType(inType) => - attr.name match { - case "first" => arrayFirst(value) - case "last" => arrayLast(value) - case "size" => arraySize(value) - case "min" => arrayMin(value) - case "max" => arrayMax(value) - } - case _: BytesType => - attr.name match { - case "first" => bytesFirst(value) - case "last" => bytesLast(value) - case "size" => bytesSize(value) - } - case KaitaiStreamType => - attr.name match { - case "size" => kaitaiStreamSize(value) - case "eof" => kaitaiStreamEof(value) - case "pos" => kaitaiStreamPos(value) - } - case et: EnumType => - attr.name match { - case "to_i" => enumToInt(value, et) - case _ => throw new TypeMismatchError(s"called invalid attribute '${attr.name}' on expression of type $valType") - } - case _: BooleanType => - attr.name match { - case "to_i" => boolToInt(value) - case _ => throw new TypeMismatchError(s"called invalid attribute '${attr.name}' on expression of type $valType") - } - } - case Ast.expr.Call(func: Ast.expr, args: Seq[Ast.expr]) => - func match { - case Ast.expr.Attribute(obj: Ast.expr, methodName: Ast.identifier) => - val objType = detectType(obj) - (objType, methodName.name) match { - // TODO: check argument quantity - case (_: StrType, "substring") => strSubstring(obj, args(0), args(1)) - case (_: StrType, "to_i") => strToInt(obj, args(0)) - case (_: BytesType, "to_s") => bytesToStr(translate(obj), args(0)) - case (_: StrType, "to_b") => strToBytes(translate(obj), args(0)) - case _ => throw new TypeMismatchError(s"don't know how to call method '$methodName' of object type '$objType'") - } - } + case call: Ast.expr.Attribute => + translateAttribute(call) + case call: Ast.expr.Call => + translateCall(call) case Ast.expr.List(values: Seq[Ast.expr]) => - val t = detectArrayType(values) - t match { - case Int1Type(_) => - val literalBytes: Seq[Byte] = values.map { - case Ast.expr.IntNum(x) => - if (x < 0 || x > 0xff) { - throw new TypeMismatchError(s"got a weird byte value in byte array: $x") - } else { - x.toByte - } - case n => - throw new TypeMismatchError(s"got $n in byte array, unable to put it literally") - } - doByteArrayLiteral(literalBytes) - case _ => - doArrayLiteral(t, values) - } - case Ast.expr.CastToType(value, typeName) => - doCast(value, typeName.name) - } - } - - def numericBinOp(left: Ast.expr, op: Ast.operator, right: Ast.expr) = { - s"(${translate(left)} ${binOp(op)} ${translate(right)})" - } - - def binOp(op: Ast.operator): String = { - op match { - case Ast.operator.Add => "+" - case Ast.operator.Sub => "-" - case Ast.operator.Mult => "*" - case Ast.operator.Div => "/" - case Ast.operator.Mod => "%" - case Ast.operator.BitAnd => "&" - case Ast.operator.BitOr => "|" - case Ast.operator.BitXor => "^" - case Ast.operator.LShift => "<<" - case Ast.operator.RShift => ">>" - } - } - - def doNumericCompareOp(left: Ast.expr, op: Ast.cmpop, right: Ast.expr): String = - s"${translate(left)} ${cmpOp(op)} ${translate(right)}" - - def doStrCompareOp(left: Ast.expr, op: Ast.cmpop, right: Ast.expr): String = - s"${translate(left)} ${cmpOp(op)} ${translate(right)}" - - def doEnumCompareOp(left: Ast.expr, op: Ast.cmpop, right: Ast.expr): String = - s"${translate(left)} ${cmpOp(op)} ${translate(right)}" - - def doBytesCompareOp(left: Ast.expr, op: Ast.cmpop, right: Ast.expr): String = - s"${translate(left)} ${cmpOp(op)} ${translate(right)}" - - def cmpOp(op: Ast.cmpop): String = { - op match { - case Ast.cmpop.Lt => "<" - case Ast.cmpop.LtE => "<=" - case Ast.cmpop.Gt => ">" - case Ast.cmpop.GtE => ">=" - case Ast.cmpop.Eq => "==" - case Ast.cmpop.NotEq => "!=" + doGuessArrayLiteral(values) + case ctt: Ast.expr.CastToType => + doCastOrArray(ctt) + case Ast.expr.ByteSizeOfType(typeName) => + doByteSizeOfType(typeName) + case Ast.expr.BitSizeOfType(typeName) => + doBitSizeOfType(typeName) } } - def doBooleanOp(op: Ast.boolop, values: Seq[Ast.expr]): String = { - val opStr = s"${booleanOp(op)}" - val dividerStr = s") $opStr (" - val valuesStr = values.map(translate).mkString("(", dividerStr, ")") - - // Improve compatibility for statements like: ( ... && ... || ... ) ? ... : ... - s" ($valuesStr) " - } - - def booleanOp(op: Ast.boolop): String = op match { - case Ast.boolop.Or => "||" - case Ast.boolop.And => "&&" - } - - def unaryOp(op: Ast.unaryop): String = op match { - case Ast.unaryop.Invert => "~" - case Ast.unaryop.Minus => "-" - case Ast.unaryop.Not => "!" - } - - def doSubscript(container: Ast.expr, idx: Ast.expr): String def doIfExp(condition: Ast.expr, ifTrue: Ast.expr, ifFalse: Ast.expr): String - def doCast(value: Ast.expr, typeName: String): String = translate(value) - - // Literals - def doIntLiteral(n: BigInt): String = n.toString - def doFloatLiteral(n: Any): String = n.toString + def doCast(value: Ast.expr, typeName: DataType): String = translate(value) + def doByteSizeOfType(typeName: Ast.typeId): String = doIntLiteral( + CommonSizeOf.bitToByteSize( + CommonSizeOf.getBitsSizeOfType( + typeName.nameAsStr, detectCastType(typeName) + ) + ) + ) + def doBitSizeOfType(typeName: Ast.typeId): String = doIntLiteral( + CommonSizeOf.getBitsSizeOfType( + typeName.nameAsStr, detectCastType(typeName) + ) + ) + def byteSizeOfValue(attrName: String, valType: DataType): String = doIntLiteral( + CommonSizeOf.bitToByteSize( + CommonSizeOf.getBitsSizeOfType(attrName, valType) + ) + ) + def byteSizeOfClassSpec(cs: ClassSpec): String = + doIntLiteral(CommonSizeOf.getByteSizeOfClassSpec(cs)) - def doStringLiteral(s: String): String = { - val encoded = s.toCharArray.map((code) => - if (code <= 0xff) { - strLiteralAsciiChar(code) - } else { - strLiteralUnicode(code) - } - ).mkString - "\"" + encoded + "\"" - } - def doBoolLiteral(n: Boolean): String = n.toString def doArrayLiteral(t: DataType, value: Seq[Ast.expr]): String = "[" + value.map((v) => translate(v)).mkString(", ") + "]" def doByteArrayLiteral(arr: Seq[Byte]): String = "[" + arr.map(_ & 0xff).mkString(", ") + "]" - - /** - * Handle ASCII character conversion for inlining into string literals. - * Default implementation consults [[asciiCharQuoteMap]] first, then - * just dumps it as is if it's a printable ASCII charcter, or calls - * [[strLiteralGenericCC]] if it's a control character. - * @param code character code to convert into string for inclusion in - * a string literal - */ - def strLiteralAsciiChar(code: Char): String = { - asciiCharQuoteMap.get(code) match { - case Some(encoded) => encoded - case None => - if (code >= 0x20 && code < 0x80) { - Character.toString(code) - } else { - strLiteralGenericCC(code) - } - } - } - - /** - * Converts generic control character code into something that's allowed - * inside a string literal. Default implementation uses octal encoding, - * which is ok for most C-derived languages. - * - * Note that we use strictly 3 octal digits to work around potential - * problems with following decimal digits, i.e. "\0" + "2" that would be - * parsed as single character "\02" = "\x02", instead of two characters - * "\x00\x32". - * @param code character code to represent - * @return string literal representation of given code - */ - def strLiteralGenericCC(code: Char): String = - "\\%03o".format(code.toInt) - - /** - * Converts Unicode (typically, non-ASCII) character code into something - * that's allowed inside a string literal. Default implementation uses - * Unicode 4-digit hex encoding, which is ok for most C-derived languages. - * @param code character code to represent - * @return string literal representation of given code - */ - def strLiteralUnicode(code: Char): String = - "\\u%04x".format(code.toInt) - - /** - * Character quotation map for inclusion in string literals. - * Default implementation includes bare minimum that seems - * to be available in all languages. - */ - val asciiCharQuoteMap: Map[Char, String] = Map( - '\t' -> "\\t", - '\n' -> "\\n", - '\r' -> "\\r", - '"' -> "\\\"", - '\\' -> "\\\\" - ) + def doByteArrayNonLiteral(elts: Seq[Ast.expr]): String = ??? def doLocalName(s: String): String = doName(s) def doName(s: String): String - def userTypeField(value: Ast.expr, attrName: String): String = - s"${translate(value)}.${doName(attrName)}" + def userTypeField(userType: UserType, value: Ast.expr, attrName: String): String = + anyField(value, attrName) + def doEnumByLabel(enumTypeAbs: List[String], label: String): String def doEnumById(enumTypeAbs: List[String], id: String): String // Predefined methods of various types def strConcat(left: Ast.expr, right: Ast.expr): String = s"${translate(left)} + ${translate(right)}" - def strToInt(s: Ast.expr, base: Ast.expr): String - def enumToInt(value: Ast.expr, et: EnumType): String def boolToInt(value: Ast.expr): String = doIfExp(value, Ast.expr.IntNum(1), Ast.expr.IntNum(0)) - def intToStr(i: Ast.expr, base: Ast.expr): String - def bytesToStr(bytesExpr: String, encoding: Ast.expr): String - def bytesFirst(b: Ast.expr): String = ??? - def bytesLast(b: Ast.expr): String = ??? - def bytesSize(b: Ast.expr): String = ??? - def strToBytes(strExpr: String, encoding: Ast.expr): String = ??? - def strLength(s: Ast.expr): String - def strReverse(s: Ast.expr): String - def strSubstring(s: Ast.expr, from: Ast.expr, to: Ast.expr): String - def arrayFirst(a: Ast.expr): String - def arrayLast(a: Ast.expr): String - def arraySize(a: Ast.expr): String - def arrayMin(a: Ast.expr): String - def arrayMax(a: Ast.expr): String + def kaitaiStreamSize(value: Ast.expr): String = anyField(value, "size") + def kaitaiStreamEof(value: Ast.expr): String = anyField(value, "is_eof") + def kaitaiStreamPos(value: Ast.expr): String = anyField(value, "pos") - def kaitaiStreamSize(value: Ast.expr): String = userTypeField(value, "size") - def kaitaiStreamEof(value: Ast.expr): String = userTypeField(value, "is_eof") - def kaitaiStreamPos(value: Ast.expr): String = userTypeField(value, "pos") + // Special convenience definition method + helper + override def bytesToStr(value: Ast.expr, expr: Ast.expr): String = + bytesToStr(translate(value), expr) + def bytesToStr(value: String, expr: Ast.expr): String + + // Helper that does simple "one size fits all" attribute calling, if it is useful + // for the language + def anyField(value: Ast.expr, attrName: String): String = + s"${translate(value)}.${doName(attrName)}" } diff --git a/shared/src/main/scala/io/kaitai/struct/translators/ByteArraysAsTrueArrays.scala b/shared/src/main/scala/io/kaitai/struct/translators/ByteArraysAsTrueArrays.scala new file mode 100644 index 000000000..bfe89886b --- /dev/null +++ b/shared/src/main/scala/io/kaitai/struct/translators/ByteArraysAsTrueArrays.scala @@ -0,0 +1,17 @@ +package io.kaitai.struct.translators + +import io.kaitai.struct.exprlang.Ast + +/** + * Provides default implementations for byte arrays operations equal to true array operations. + * Useful for languages which do not make (major) distinctions between true arrays and byte arrays. + * @tparam T translation result + */ +trait ByteArraysAsTrueArrays[T] extends CommonMethods[T] { + override def bytesSubscript(container: Ast.expr, idx: Ast.expr): T = arraySubscript(container, idx) + override def bytesFirst(b: Ast.expr): T = arrayFirst(b) + override def bytesLast(b: Ast.expr): T = arrayLast(b) + override def bytesLength(b: Ast.expr): T = arraySize(b) + override def bytesMin(b: Ast.expr): T = arrayMin(b) + override def bytesMax(b: Ast.expr): T = arrayMax(b) +} diff --git a/shared/src/main/scala/io/kaitai/struct/translators/CSharpTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/CSharpTranslator.scala index 997d589d5..25027661c 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/CSharpTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/CSharpTranslator.scala @@ -1,13 +1,14 @@ package io.kaitai.struct.translators -import io.kaitai.struct.Utils +import io.kaitai.struct.{ImportList, Utils} import io.kaitai.struct.datatype.DataType import io.kaitai.struct.datatype.DataType._ import io.kaitai.struct.exprlang.Ast import io.kaitai.struct.exprlang.Ast._ +import io.kaitai.struct.format.Identifier import io.kaitai.struct.languages.CSharpCompiler -class CSharpTranslator(provider: TypeProvider) extends BaseTranslator(provider) { +class CSharpTranslator(provider: TypeProvider, importList: ImportList) extends BaseTranslator(provider) { override def doArrayLiteral(t: DataType, value: Seq[expr]): String = { val nativeType = CSharpCompiler.kaitaiType2NativeType(t) val commaStr = value.map((v) => translate(v)).mkString(", ") @@ -16,6 +17,8 @@ class CSharpTranslator(provider: TypeProvider) extends BaseTranslator(provider) override def doByteArrayLiteral(arr: Seq[Byte]): String = s"new byte[] { ${arr.map(_ & 0xff).mkString(", ")} }" + override def doByteArrayNonLiteral(elts: Seq[Ast.expr]): String = + s"new byte[] { ${elts.map(translate).mkString(", ")} }" override val asciiCharQuoteMap: Map[Char, String] = Map( '\t' -> "\\t", @@ -43,10 +46,15 @@ class CSharpTranslator(provider: TypeProvider) extends BaseTranslator(provider) } override def doName(s: String) = - if (s.startsWith("_")) - s"M${Utils.upperCamelCase(s)}" - else + if (s.startsWith("_")) { + s match { + case Identifier.SWITCH_ON => "on" + case Identifier.INDEX => "i" + case _ => s"M${Utils.upperCamelCase(s)}" + } + } else { s"${Utils.upperCamelCase(s)}" + } override def doEnumByLabel(enumTypeAbs: List[String], label: String): String = s"${enumClass(enumTypeAbs)}.${Utils.upperCamelCase(label)}" @@ -71,20 +79,26 @@ class CSharpTranslator(provider: TypeProvider) extends BaseTranslator(provider) override def doBytesCompareOp(left: Ast.expr, op: Ast.cmpop, right: Ast.expr): String = s"(${CSharpCompiler.kstreamName}.ByteArrayCompare(${translate(left)}, ${translate(right)}) ${cmpOp(op)} 0)" - override def doSubscript(container: expr, idx: expr): String = + override def arraySubscript(container: expr, idx: expr): String = s"${translate(container)}[${translate(idx)}]" override def doIfExp(condition: expr, ifTrue: expr, ifFalse: expr): String = s"(${translate(condition)} ? ${translate(ifTrue)} : ${translate(ifFalse)})" - override def doCast(value: Ast.expr, typeName: String): String = - s"((${Utils.upperCamelCase(typeName)}) (${translate(value)}))" + override def doCast(value: Ast.expr, typeName: DataType): String = + s"((${CSharpCompiler.kaitaiType2NativeType(typeName)}) (${translate(value)}))" // Predefined methods of various types - override def strToInt(s: expr, base: expr): String = + override def strToInt(s: expr, base: expr): String = { + importList.add("System") s"Convert.ToInt64(${translate(s)}, ${translate(base)})" + } override def enumToInt(v: expr, et: EnumType): String = translate(v) - override def intToStr(i: expr, base: expr): String = - s"Convert.ToString(${translate(i)}, ${translate(base)})" + override def floatToInt(v: expr): String = + s"(long) (${translate(v)})" + override def intToStr(i: expr, base: expr): String = { + importList.add("System") + s"Convert.ToString((long) (${translate(i)}), ${translate(base)})" + } override def bytesToStr(bytesExpr: String, encoding: Ast.expr): String = s"System.Text.Encoding.GetEncoding(${translate(encoding)}).GetString($bytesExpr)" override def strLength(s: expr): String = @@ -97,6 +111,8 @@ class CSharpTranslator(provider: TypeProvider) extends BaseTranslator(provider) override def strSubstring(s: expr, from: expr, to: expr): String = s"${translate(s)}.Substring(${translate(from)}, ${translate(to)} - ${translate(from)})" + override def strToBytes(s: expr, encoding: expr): String = + "" // TODO: implement override def arrayFirst(a: expr): String = s"${translate(a)}[0]" @@ -106,8 +122,12 @@ class CSharpTranslator(provider: TypeProvider) extends BaseTranslator(provider) } override def arraySize(a: expr): String = s"${translate(a)}.Count" - override def arrayMin(a: Ast.expr): String = + override def arrayMin(a: Ast.expr): String = { + importList.add("System.Linq") s"${translate(a)}.Min()" - override def arrayMax(a: Ast.expr): String = + } + override def arrayMax(a: Ast.expr): String = { + importList.add("System.Linq") s"${translate(a)}.Max()" + } } diff --git a/shared/src/main/scala/io/kaitai/struct/translators/CommonArraysAndCast.scala b/shared/src/main/scala/io/kaitai/struct/translators/CommonArraysAndCast.scala new file mode 100644 index 000000000..945ec52b8 --- /dev/null +++ b/shared/src/main/scala/io/kaitai/struct/translators/CommonArraysAndCast.scala @@ -0,0 +1,101 @@ +package io.kaitai.struct.translators + +import io.kaitai.struct.datatype.DataType +import io.kaitai.struct.datatype.DataType._ +import io.kaitai.struct.exprlang.Ast +import io.kaitai.struct.precompile.TypeMismatchError + +/** + * Common implementation of arrays translations: + * + * * type guessing + * * type enforcing with a cast + * * rendering of byte arrays and true arrays + * * call to actual casting implementation + * + * @tparam T translation result type + */ +trait CommonArraysAndCast[T] extends TypeDetector { + /** + * Processes elements inside a given [[Ast.expr.List]] element to render them + * as either byte array literal or true array. + * @param values elements from a list + * @return translation result + */ + def doGuessArrayLiteral(values: Seq[Ast.expr]): T = { + val elementType = detectArrayType(values) + elementType match { + case Int1Type(_) => + val literalBytes: Seq[Byte] = values.map { + case Ast.expr.IntNum(x) => + if (x < 0 || x > 0xff) { + throw new TypeMismatchError(s"got a weird byte value in byte array: $x") + } else { + x.toByte + } + case n => + throw new TypeMismatchError(s"got $n in byte array, unable to put it literally") + } + doByteArrayLiteral(literalBytes) + case _ => + doArrayLiteral(elementType, values) + } + } + + /** + * Processes an [[Ast.expr.CastToType]] element, by checking if + * this is an literal array type enforcement cast first, and + * rendering it accordingly as proper literal, or invoking + * the normal [[doCast]] otherwise. + * @param v CastToType element + * @return translation result + */ + def doCastOrArray(v: Ast.expr.CastToType): T = { + val castToType = detectCastType(v.typeName) + + v.value match { + case array: Ast.expr.List => + // Special handling for literal arrays: if cast is present, + // then we don't need to guess the data type + castToType match { + case _: BytesType => + doByteArray(array.elts) + case ArrayType(elType) => + doArrayLiteral(elType, array.elts) + case _ => + // No luck, this is some kind of weird cast, not a type enforcement; + // Just do it and let real type casting deal with it. + doCast(v.value, castToType) + } + case _ => + doCast(v.value, castToType) + } + } + + def doCast(value: Ast.expr, typeName: DataType): T + def doArrayLiteral(t: DataType, value: Seq[Ast.expr]): T + def doByteArrayLiteral(arr: Seq[Byte]): T + def doByteArrayNonLiteral(elts: Seq[Ast.expr]): T + + private def doByteArray(elts: Seq[Ast.expr]): T = { + valuesAsByteArrayLiteral(elts) match { + case Some(arr) => + doByteArrayLiteral(arr) + case None => + doByteArrayNonLiteral(elts) + } + } + + private def valuesAsByteArrayLiteral(elts: Seq[Ast.expr]): Option[Seq[Byte]] = { + Some(elts.map { + case Ast.expr.IntNum(x) => + if (x < 0 || x > 0xff) { + return None + } else { + x.toByte + } + case _ => + return None + }) + } +} diff --git a/shared/src/main/scala/io/kaitai/struct/translators/CommonLiterals.scala b/shared/src/main/scala/io/kaitai/struct/translators/CommonLiterals.scala new file mode 100644 index 000000000..29ee8ee4a --- /dev/null +++ b/shared/src/main/scala/io/kaitai/struct/translators/CommonLiterals.scala @@ -0,0 +1,80 @@ +package io.kaitai.struct.translators + +/** + * Implementations of translations of literals in C-style, common to many + * languages. + */ +trait CommonLiterals { + def doIntLiteral(n: BigInt): String = n.toString + def doFloatLiteral(n: Any): String = n.toString + + def doStringLiteral(s: String): String = { + val encoded = s.toCharArray.map((code) => + if (code <= 0xff) { + strLiteralAsciiChar(code) + } else { + strLiteralUnicode(code) + } + ).mkString + "\"" + encoded + "\"" + } + def doBoolLiteral(n: Boolean): String = n.toString + + /** + * Handle ASCII character conversion for inlining into string literals. + * Default implementation consults [[asciiCharQuoteMap]] first, then + * just dumps it as is if it's a printable ASCII charcter, or calls + * [[strLiteralGenericCC]] if it's a control character. + * @param code character code to convert into string for inclusion in + * a string literal + */ + def strLiteralAsciiChar(code: Char): String = { + asciiCharQuoteMap.get(code) match { + case Some(encoded) => encoded + case None => + if (code >= 0x20 && code < 0x80) { + Character.toString(code) + } else { + strLiteralGenericCC(code) + } + } + } + + /** + * Converts generic control character code into something that's allowed + * inside a string literal. Default implementation uses octal encoding, + * which is ok for most C-derived languages. + * + * Note that we use strictly 3 octal digits to work around potential + * problems with following decimal digits, i.e. "\0" + "2" that would be + * parsed as single character "\02" = "\x02", instead of two characters + * "\x00\x32". + * @param code character code to represent + * @return string literal representation of given code + */ + def strLiteralGenericCC(code: Char): String = + "\\%03o".format(code.toInt) + + /** + * Converts Unicode (typically, non-ASCII) character code into something + * that's allowed inside a string literal. Default implementation uses + * Unicode 4-digit hex encoding, which is ok for most C-derived languages. + * @param code character code to represent + * @return string literal representation of given code + */ + def strLiteralUnicode(code: Char): String = + "\\u%04x".format(code.toInt) + + /** + * Character quotation map for inclusion in string literals. + * Default implementation includes bare minimum that seems + * to be available in all languages. + */ + val asciiCharQuoteMap: Map[Char, String] = Map( + '\t' -> "\\t", + '\n' -> "\\n", + '\r' -> "\\r", + '"' -> "\\\"", + '\\' -> "\\\\" + ) +} diff --git a/shared/src/main/scala/io/kaitai/struct/translators/CommonMethods.scala b/shared/src/main/scala/io/kaitai/struct/translators/CommonMethods.scala new file mode 100644 index 000000000..e4c12e63b --- /dev/null +++ b/shared/src/main/scala/io/kaitai/struct/translators/CommonMethods.scala @@ -0,0 +1,140 @@ +package io.kaitai.struct.translators + +import io.kaitai.struct.datatype.DataType +import io.kaitai.struct.datatype.DataType._ +import io.kaitai.struct.exprlang.Ast +import io.kaitai.struct.format.Identifier +import io.kaitai.struct.precompile.TypeMismatchError + +abstract trait CommonMethods[T] extends TypeDetector { + /** + * Translates a certain attribute call (as in `foo.bar`) into a rendition + * of expression in certain target language. + * @note Must be kept in sync with [[TypeDetector.detectAttributeType]] + * @param call attribute call expression to translate + * @return result of translation as [[T]] + */ + def translateAttribute(call: Ast.expr.Attribute): T = { + val attr = call.attr + val value = call.value + val valType = detectType(value) + + // Special case: will be compiled as compile-time determined constant + if (attr.name == Identifier.SIZEOF) + return byteSizeOfValue(value.toString, valType) + + valType match { + case ut: UserType => + userTypeField(ut, value, attr.name) + case _: BytesType => + attr.name match { + case "first" => bytesFirst(value) + case "last" => bytesLast(value) + case "length" | "size" => bytesLength(value) + case "min" => bytesMin(value) + case "max" => bytesMax(value) + } + case _: StrType => + attr.name match { + case "length" => strLength(value) + case "reverse" => strReverse(value) + case "to_i" => strToInt(value, Ast.expr.IntNum(10)) + } + case _: IntType => + attr.name match { + case "to_s" => intToStr(value, Ast.expr.IntNum(10)) + } + case _: FloatType => + attr.name match { + case "to_i" => floatToInt(value) + } + case ArrayType(inType) => + attr.name match { + case "first" => arrayFirst(value) + case "last" => arrayLast(value) + case "size" => arraySize(value) + case "min" => arrayMin(value) + case "max" => arrayMax(value) + } + case KaitaiStreamType => + attr.name match { + case "size" => kaitaiStreamSize(value) + case "eof" => kaitaiStreamEof(value) + case "pos" => kaitaiStreamPos(value) + } + case et: EnumType => + attr.name match { + case "to_i" => enumToInt(value, et) + case _ => throw new TypeMismatchError(s"called invalid attribute '${attr.name}' on expression of type $valType") + } + case _: BooleanType => + attr.name match { + case "to_i" => boolToInt(value) + case _ => throw new TypeMismatchError(s"called invalid attribute '${attr.name}' on expression of type $valType") + } + } + } + + /** + * Translates a certain function call (as in `foo.bar(arg1, arg2)`) into a + * rendition of expression in certain target language. + * @note Must be kept in sync with [[TypeDetector.detectCallType]] + * @param call function call expression to translate + * @return result of translation as [[T]] + */ + def translateCall(call: Ast.expr.Call): T = { + val func = call.func + val args = call.args + + func match { + case Ast.expr.Attribute(obj: Ast.expr, methodName: Ast.identifier) => + val objType = detectType(obj) + (objType, methodName.name) match { + // TODO: check argument quantity + case (_: StrType, "substring") => strSubstring(obj, args(0), args(1)) + case (_: StrType, "to_i") => strToInt(obj, args(0)) + case (_: StrType, "to_b") => strToBytes(obj, args(0)) + case (_: BytesType, "to_s") => bytesToStr(obj, args(0)) + case _ => throw new TypeMismatchError(s"don't know how to call method '$methodName' of object type '$objType'") + } + } + } + + def userTypeField(ut: UserType, value: Ast.expr, name: String): T + + def bytesSubscript(container: Ast.expr, idx: Ast.expr): T + def bytesFirst(b: Ast.expr): T + def bytesLast(b: Ast.expr): T + def bytesLength(b: Ast.expr): T + def bytesMin(b: Ast.expr): T + def bytesMax(b: Ast.expr): T + + def strLength(s: Ast.expr): T + def strReverse(s: Ast.expr): T + def strToInt(s: Ast.expr, base: Ast.expr): T + def strSubstring(s: Ast.expr, from: Ast.expr, to: Ast.expr): T + def strToBytes(s: Ast.expr, encoding: Ast.expr): T + + def bytesToStr(value: Ast.expr, expr: Ast.expr): T + + def intToStr(value: Ast.expr, num: Ast.expr): T + + def floatToInt(value: Ast.expr): T + + def kaitaiStreamSize(value: Ast.expr): T + def kaitaiStreamEof(value: Ast.expr): T + def kaitaiStreamPos(value: Ast.expr): T + + def arraySubscript(container: Ast.expr, idx: Ast.expr): T + def arrayFirst(a: Ast.expr): T + def arrayLast(a: Ast.expr): T + def arraySize(a: Ast.expr): T + def arrayMin(a: Ast.expr): T + def arrayMax(a: Ast.expr): T + + def enumToInt(value: Ast.expr, et: EnumType): T + + def boolToInt(value: Ast.expr): T + + def byteSizeOfValue(attrName: String, valType: DataType): T +} diff --git a/shared/src/main/scala/io/kaitai/struct/translators/CommonOps.scala b/shared/src/main/scala/io/kaitai/struct/translators/CommonOps.scala new file mode 100644 index 000000000..a55019e55 --- /dev/null +++ b/shared/src/main/scala/io/kaitai/struct/translators/CommonOps.scala @@ -0,0 +1,67 @@ +package io.kaitai.struct.translators + +import io.kaitai.struct.exprlang.Ast + +trait CommonOps extends AbstractTranslator { + def numericBinOp(left: Ast.expr, op: Ast.operator, right: Ast.expr) = { + s"(${translate(left)} ${binOp(op)} ${translate(right)})" + } + + def binOp(op: Ast.operator): String = { + op match { + case Ast.operator.Add => "+" + case Ast.operator.Sub => "-" + case Ast.operator.Mult => "*" + case Ast.operator.Div => "/" + case Ast.operator.Mod => "%" + case Ast.operator.BitAnd => "&" + case Ast.operator.BitOr => "|" + case Ast.operator.BitXor => "^" + case Ast.operator.LShift => "<<" + case Ast.operator.RShift => ">>" + } + } + + def doNumericCompareOp(left: Ast.expr, op: Ast.cmpop, right: Ast.expr): String = + s"${translate(left)} ${cmpOp(op)} ${translate(right)}" + + def doStrCompareOp(left: Ast.expr, op: Ast.cmpop, right: Ast.expr): String = + s"${translate(left)} ${cmpOp(op)} ${translate(right)}" + + def doEnumCompareOp(left: Ast.expr, op: Ast.cmpop, right: Ast.expr): String = + s"${translate(left)} ${cmpOp(op)} ${translate(right)}" + + def doBytesCompareOp(left: Ast.expr, op: Ast.cmpop, right: Ast.expr): String = + s"${translate(left)} ${cmpOp(op)} ${translate(right)}" + + def cmpOp(op: Ast.cmpop): String = { + op match { + case Ast.cmpop.Lt => "<" + case Ast.cmpop.LtE => "<=" + case Ast.cmpop.Gt => ">" + case Ast.cmpop.GtE => ">=" + case Ast.cmpop.Eq => "==" + case Ast.cmpop.NotEq => "!=" + } + } + + def doBooleanOp(op: Ast.boolop, values: Seq[Ast.expr]): String = { + val opStr = s"${booleanOp(op)}" + val dividerStr = s") $opStr (" + val valuesStr = values.map(translate).mkString("(", dividerStr, ")") + + // Improve compatibility for statements like: ( ... && ... || ... ) ? ... : ... + s" ($valuesStr) " + } + + def booleanOp(op: Ast.boolop): String = op match { + case Ast.boolop.Or => "||" + case Ast.boolop.And => "&&" + } + + def unaryOp(op: Ast.unaryop): String = op match { + case Ast.unaryop.Invert => "~" + case Ast.unaryop.Minus => "-" + case Ast.unaryop.Not => "!" + } +} diff --git a/shared/src/main/scala/io/kaitai/struct/translators/CommonSizeOf.scala b/shared/src/main/scala/io/kaitai/struct/translators/CommonSizeOf.scala new file mode 100644 index 000000000..a7f91f6d7 --- /dev/null +++ b/shared/src/main/scala/io/kaitai/struct/translators/CommonSizeOf.scala @@ -0,0 +1,40 @@ +package io.kaitai.struct.translators + +import io.kaitai.struct.datatype.DataType +import io.kaitai.struct.datatype.DataType.CalcUserType +import io.kaitai.struct.format.{ClassSpec, DynamicSized, FixedSized} +import io.kaitai.struct.precompile.{CalculateSeqSizes, InternalCompilerError, TypeMismatchError} + +object CommonSizeOf { + /** + * Converts bit size to byte size, rounded up to byte margin. + * @param bits number of bits + * @return number of bytes it takes + */ + def bitToByteSize(bits: Int): Int = (bits + 7) / 8 + + def getBitsSizeOfType(typeName: String, dataType: DataType): Int = { + val sizeSpec = dataType match { + case cut: CalcUserType => cut.classSpec.get.seqSize + case _ => CalculateSeqSizes.dataTypeBitsSize(dataType) + } + sizeSpec match { + case FixedSized(n) => + n + case DynamicSized => + throw new TypeMismatchError(s"unable to derive sizeof of `$typeName`: dynamic sized type") + case other => + throw InternalCompilerError(s"internal compiler error: sizeSpec=$other for `$typeName`") + } + } + + def getByteSizeOfClassSpec(cs: ClassSpec): Int = { + bitToByteSize(cs.seqSize match { + case FixedSized(n) => n + case DynamicSized => + throw new TypeMismatchError(s"unable to derive sizeof for type `${cs.nameAsStr}`: dynamic sized type") + case other => + throw InternalCompilerError(s"internal compiler error: sizeSpec=$other for type `${cs.nameAsStr}`") + }) + } +} diff --git a/shared/src/main/scala/io/kaitai/struct/translators/ConstructTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/ConstructTranslator.scala new file mode 100644 index 000000000..1476d214c --- /dev/null +++ b/shared/src/main/scala/io/kaitai/struct/translators/ConstructTranslator.scala @@ -0,0 +1,31 @@ +package io.kaitai.struct.translators + +import io.kaitai.struct.ImportList +import io.kaitai.struct.exprlang.Ast +import io.kaitai.struct.format.Identifier + +class ConstructTranslator(provider: TypeProvider, importList: ImportList) extends PythonTranslator(provider, importList) { + override def doLocalName(s: String) = { + s match { + case Identifier.ITERATOR => "obj_" + case Identifier.INDEX => "i" + case Identifier.ROOT => "this._root" + case Identifier.IO => "_io" + case _ => s"this.${doName(s)}" + } + } + + override def doName(s: String) = { + s match { + case Identifier.PARENT => "_" + case _ => s + } + } + + override def kaitaiStreamSize(value: Ast.expr): String = + s"stream_size(${translate(value)})" + override def kaitaiStreamEof(value: Ast.expr): String = + s"stream_iseof(${translate(value)})" + override def kaitaiStreamPos(value: Ast.expr): String = + s"stream_tell(${translate(value)})" +} diff --git a/shared/src/main/scala/io/kaitai/struct/translators/CppTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/CppTranslator.scala index 699a63dc1..d39c3eaee 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/CppTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/CppTranslator.scala @@ -2,17 +2,68 @@ package io.kaitai.struct.translators import java.nio.charset.Charset -import io.kaitai.struct.Utils -import io.kaitai.struct.exprlang.Ast -import io.kaitai.struct.exprlang.Ast.expr +import io.kaitai.struct.CppRuntimeConfig.{RawPointers, SharedPointers, UniqueAndRawPointers} import io.kaitai.struct.datatype.DataType import io.kaitai.struct.datatype.DataType._ +import io.kaitai.struct.exprlang.Ast +import io.kaitai.struct.exprlang.Ast.expr import io.kaitai.struct.format.Identifier import io.kaitai.struct.languages.CppCompiler +import io.kaitai.struct.{ImportList, RuntimeConfig, Utils} -class CppTranslator(provider: TypeProvider) extends BaseTranslator(provider) { +class CppTranslator(provider: TypeProvider, importListSrc: ImportList, config: RuntimeConfig) extends BaseTranslator(provider) { val CHARSET_UTF8 = Charset.forName("UTF-8") + /** + * Handles integer literals for C++ by appending relevant suffix to + * decimal notation. + * + * Note that suffixes essentially mean "long", "unsigned long", + * and "unsigned long long", which are not really guaranteed to match + * `int32_t`, `uint32_t` and `uint64_t`, but it would work for majority + * of current compilers. + * + * For reference, ranges of integers that are used in this conversion are: + * + * * int32_t (no suffix): –2147483648..2147483647 + * * uint32_t (UL): 0..4294967295 + * * int64_t (LL): -9223372036854775808..9223372036854775807 + * * uint64_t (ULL): 0..18446744073709551615 + * + * Merging all these ranges, we get the following decision tree: + * + * * -9223372036854775808..-2147483649 => LL + * * -2147483648..2147483647 => no suffix + * * 2147483648..4294967295 => UL + * * 4294967296..9223372036854775807 => LL + * * 9223372036854775808..18446744073709551615 => ULL + * + * Beyond these boundaries, C++ is unlikely to be able to represent + * these anyway, so we just drop the suffix and hope for the miracle. + * + * @param n integer to render + * @return rendered integer literal in C++ syntax as string + */ + override def doIntLiteral(n: BigInt): String = { + val suffix = if (n < -9223372036854775808L) { + "" // too low, no suffix would help anyway + } else if (n <= -2147483649L) { + "LL" // -9223372036854775808..–2147483649 + } else if (n <= 2147483647L) { + "" // -2147483648..2147483647 + } else if (n <= 4294967295L) { + "UL" // 2147483648..4294967295 + } else if (n <= 9223372036854775807L) { + "LL" // 4294967296..9223372036854775807 + } else if (n <= Utils.MAX_UINT64) { + "ULL" // 9223372036854775808..18446744073709551615 + } else { + "" // too high, no suffix would help anyway + } + + s"$n$suffix" + } + /** * Handles string literal for C++ by wrapping a C `const char*`-style string * into a std::string constructor. Note that normally std::string @@ -66,17 +117,19 @@ class CppTranslator(provider: TypeProvider) extends BaseTranslator(provider) { } } - override def userTypeField(value: expr, attrName: String): String = + override def anyField(value: expr, attrName: String): String = s"${translate(value)}->${doName(attrName)}" override def doName(s: String) = s match { case Identifier.ITERATOR => "_" case Identifier.ITERATOR2 => "_buf" + case Identifier.INDEX => "i" case _ => s"$s()" } override def doEnumByLabel(enumType: List[String], label: String): String = - (enumType.last + "_" + label).toUpperCase + CppCompiler.types2class(enumType.dropRight(1)) + "::" + + (enumType.last + "_" + label).toUpperCase override def doEnumById(enumType: List[String], id: String): String = s"static_cast<${CppCompiler.types2class(enumType)}>($id)" @@ -90,12 +143,24 @@ class CppTranslator(provider: TypeProvider) extends BaseTranslator(provider) { } } - override def doSubscript(container: expr, idx: expr): String = + override def arraySubscript(container: expr, idx: expr): String = s"${translate(container)}->at(${translate(idx)})" override def doIfExp(condition: expr, ifTrue: expr, ifFalse: expr): String = s"((${translate(condition)}) ? (${translate(ifTrue)}) : (${translate(ifFalse)}))" - override def doCast(value: Ast.expr, typeName: String): String = - s"static_cast<${CppCompiler.types2class(List(typeName))}*>(${translate(value)})" + override def doCast(value: Ast.expr, typeName: DataType): String = + config.cppConfig.pointers match { + case RawPointers | UniqueAndRawPointers => + cppStaticCast(value, typeName) + case SharedPointers => + typeName match { + case ut: UserType => + s"std::static_pointer_cast<${CppCompiler.types2class(ut.classSpec.get.name)}>(${translate(value)})" + case _ => cppStaticCast(value, typeName) + } + } + + def cppStaticCast(value: Ast.expr, typeName: DataType): String = + s"static_cast<${CppCompiler.kaitaiType2NativeType(config.cppConfig, typeName)}>(${translate(value)})" // Predefined methods of various types override def strToInt(s: expr, base: expr): String = { @@ -108,7 +173,9 @@ class CppTranslator(provider: TypeProvider) extends BaseTranslator(provider) { override def enumToInt(v: expr, et: EnumType): String = translate(v) override def boolToInt(v: expr): String = - translate(v) + s"((${translate(v)}) ? 1 : 0)" + override def floatToInt(v: expr): String = + s"static_cast(${translate(v)})" override def intToStr(i: expr, base: expr): String = { val baseStr = translate(base) baseStr match { @@ -121,12 +188,36 @@ class CppTranslator(provider: TypeProvider) extends BaseTranslator(provider) { } override def bytesToStr(bytesExpr: String, encoding: Ast.expr): String = s"${CppCompiler.kstreamName}::bytes_to_str($bytesExpr, ${translate(encoding)})" + override def bytesLength(b: Ast.expr): String = + s"${translate(b)}.length()" + + override def bytesSubscript(container: Ast.expr, idx: Ast.expr): String = + s"${translate(container)}[${translate(idx)}]" + override def bytesFirst(b: Ast.expr): String = { + config.cppConfig.stdStringFrontBack match { + case true => s"${translate(b)}.front()" + case false => s"${translate(b)}[0]" + } + } + override def bytesLast(b: Ast.expr): String = { + config.cppConfig.stdStringFrontBack match { + case true => s"${translate(b)}.back()" + case false => s"${translate(b)}[${translate(b)}.length() - 1]" + } + } + override def bytesMin(b: Ast.expr): String = + s"${CppCompiler.kstreamName}::byte_array_min(${translate(b)})" + override def bytesMax(b: Ast.expr): String = + s"${CppCompiler.kstreamName}::byte_array_max(${translate(b)})" + override def strLength(s: expr): String = s"${translate(s)}.length()" override def strReverse(s: expr): String = s"${CppCompiler.kstreamName}::reverse(${translate(s)})" override def strSubstring(s: expr, from: expr, to: expr): String = s"${translate(s)}.substr(${translate(from)}, (${translate(to)}) - (${translate(from)}))" + override def strToBytes(s: expr, encoding: expr): String = + "" // TODO: implement override def arrayFirst(a: expr): String = s"${translate(a)}->front()" @@ -135,10 +226,12 @@ class CppTranslator(provider: TypeProvider) extends BaseTranslator(provider) { override def arraySize(a: expr): String = s"${translate(a)}->size()" override def arrayMin(a: expr): String = { + importListSrc.add("algorithm") val v = translate(a) s"*std::min_element($v->begin(), $v->end())" } override def arrayMax(a: expr): String = { + importListSrc.add("algorithm") val v = translate(a) s"*std::max_element($v->begin(), $v->end())" } diff --git a/shared/src/main/scala/io/kaitai/struct/translators/ExpressionValidator.scala b/shared/src/main/scala/io/kaitai/struct/translators/ExpressionValidator.scala new file mode 100644 index 000000000..f928629c7 --- /dev/null +++ b/shared/src/main/scala/io/kaitai/struct/translators/ExpressionValidator.scala @@ -0,0 +1,129 @@ +package io.kaitai.struct.translators + +import io.kaitai.struct.datatype.DataType +import io.kaitai.struct.exprlang.Ast +import io.kaitai.struct.format.Identifier + +/** + * Validates expressions usage of types (in typecasting operator, + * enums, sizeof operator, etc) to match actual layout of class specs + * loaded in type provider. + * + * Implemented essentially as a no-op translator, which does all the + * recursive traversing that regular translator performs, but outputs + * not result. + * + * @param provider TypeProvider that will answer queries on user types + */ +class ExpressionValidator(val provider: TypeProvider) + extends TypeDetector(provider) + with CommonMethods[Unit] + with ByteArraysAsTrueArrays[Unit] { + /** + * Validates one expression. Throws exceptions when encounters an error. + * @param v expression to validate + */ + def validate(v: Ast.expr): Unit = { + v match { + case _: Ast.expr.IntNum | + _: Ast.expr.FloatNum | + _: Ast.expr.Str | + _: Ast.expr.Bool => // all simple literals are good and valid + case Ast.expr.EnumById(enumType, id, inType) => + provider.resolveEnum(inType, enumType.name) + validate(id) + case Ast.expr.EnumByLabel(enumType, label, inType) => + provider.resolveEnum(inType, enumType.name) + // TODO: check that label belongs to that enum + case Ast.expr.Name(name: Ast.identifier) => + if (name.name == Identifier.SIZEOF) { + CommonSizeOf.getByteSizeOfClassSpec(provider.nowClass) + } else { + // local name already checked by type detection + } + case Ast.expr.UnaryOp(op: Ast.unaryop, inner: Ast.expr) => + validate(inner) + case Ast.expr.Compare(left: Ast.expr, op: Ast.cmpop, right: Ast.expr) => + validate(left) + validate(right) + case Ast.expr.BinOp(left: Ast.expr, op: Ast.operator, right: Ast.expr) => + validate(left) + validate(right) + case Ast.expr.BoolOp(op: Ast.boolop, values: Seq[Ast.expr]) => + values.foreach(validate) + case Ast.expr.IfExp(condition, ifTrue, ifFalse) => + validate(condition) + validate(ifTrue) + validate(ifFalse) + case Ast.expr.Subscript(container: Ast.expr, idx: Ast.expr) => + validate(container) + validate(idx) + case call: Ast.expr.Attribute => + translateAttribute(call) + case call: Ast.expr.Call => + translateCall(call) + case Ast.expr.List(values: Seq[Ast.expr]) => + values.foreach(validate) + case ctt: Ast.expr.CastToType => + validate(ctt.value) + detectCastType(ctt.typeName) + case Ast.expr.ByteSizeOfType(typeName) => + CommonSizeOf.getBitsSizeOfType(typeName.nameAsStr, detectCastType(typeName)) + case Ast.expr.BitSizeOfType(typeName) => + CommonSizeOf.getBitsSizeOfType(typeName.nameAsStr, detectCastType(typeName)) + } + } + + override def userTypeField(ut: DataType.UserType, value: Ast.expr, name: String): Unit = { + validate(value) + // TODO: check that field exists + } + + override def strLength(s: Ast.expr): Unit = validate(s) + override def strReverse(s: Ast.expr): Unit = validate(s) + override def strToInt(s: Ast.expr, base: Ast.expr): Unit = { + validate(s) + validate(base) + } + override def strSubstring(s: Ast.expr, from: Ast.expr, to: Ast.expr): Unit = { + validate(s) + validate(from) + validate(to) + } + override def strToBytes(s: Ast.expr, encoding: Ast.expr): Unit = { + // TODO: implement + } + + override def bytesToStr(value: Ast.expr, expr: Ast.expr): Unit = { + validate(value) + validate(expr) + } + + override def intToStr(value: Ast.expr, num: Ast.expr): Unit = { + validate(value) + validate(num) + } + + override def floatToInt(value: Ast.expr): Unit = validate(value) + + override def kaitaiStreamSize(value: Ast.expr): Unit = validate(value) + override def kaitaiStreamEof(value: Ast.expr): Unit = validate(value) + override def kaitaiStreamPos(value: Ast.expr): Unit = validate(value) + + override def arraySubscript(container: Ast.expr, idx: Ast.expr): Unit = { + validate(container) + validate(idx) + } + override def arrayFirst(a: Ast.expr): Unit = validate(a) + override def arrayLast(a: Ast.expr): Unit = validate(a) + override def arraySize(a: Ast.expr): Unit = validate(a) + override def arrayMin(a: Ast.expr): Unit = validate(a) + override def arrayMax(a: Ast.expr): Unit = validate(a) + + override def enumToInt(value: Ast.expr, et: DataType.EnumType): Unit = validate(value) + + override def boolToInt(value: Ast.expr): Unit = validate(value) + + override def byteSizeOfValue(attrName: String, valType: DataType): Unit = + CommonSizeOf.getBitsSizeOfType(attrName, valType) +} diff --git a/shared/src/main/scala/io/kaitai/struct/translators/GoTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/GoTranslator.scala new file mode 100644 index 000000000..50093baa9 --- /dev/null +++ b/shared/src/main/scala/io/kaitai/struct/translators/GoTranslator.scala @@ -0,0 +1,452 @@ +package io.kaitai.struct.translators + +import io.kaitai.struct.datatype.DataType +import io.kaitai.struct.datatype.DataType._ +import io.kaitai.struct.exprlang.Ast +import io.kaitai.struct.format.Identifier +import io.kaitai.struct.languages.GoCompiler +import io.kaitai.struct.precompile.TypeMismatchError +import io.kaitai.struct.{ImportList, StringLanguageOutputWriter, Utils} + +sealed trait TranslatorResult +case class ResultString(s: String) extends TranslatorResult +case class ResultLocalVar(n: Int) extends TranslatorResult + +class GoTranslator(out: StringLanguageOutputWriter, provider: TypeProvider, importList: ImportList) + extends TypeDetector(provider) + with AbstractTranslator + with CommonLiterals + with CommonOps + with CommonArraysAndCast[TranslatorResult] + with CommonMethods[TranslatorResult] + with ByteArraysAsTrueArrays[TranslatorResult] { + + import io.kaitai.struct.languages.GoCompiler._ + + var returnRes: Option[String] = None + + override def translate(v: Ast.expr): String = resToStr(translateExpr(v)) + + def resToStr(r: TranslatorResult): String = r match { + case ResultString(s) => s + case ResultLocalVar(n) => localVarName(n) + } + + def translateExpr(v: Ast.expr): TranslatorResult = { + v match { + case Ast.expr.IntNum(n) => + trIntLiteral(n) + case Ast.expr.FloatNum(n) => + trFloatLiteral(n) + case Ast.expr.Str(s) => + trStringLiteral(s) + case Ast.expr.Bool(n) => + trBoolLiteral(n) + case Ast.expr.BoolOp(op, values) => + trBooleanOp(op, values) + case Ast.expr.BinOp(left: Ast.expr, op: Ast.operator, right: Ast.expr) => + (detectType(left), detectType(right), op) match { + case (_: NumericType, _: NumericType, _) => + trNumericBinOp(left, op, right) + case (_: StrType, _: StrType, Ast.operator.Add) => + trStrConcat(left, right) + case (ltype, rtype, _) => + throw new TypeMismatchError(s"can't do $ltype $op $rtype") + } + case Ast.expr.UnaryOp(op, operand) => + ResultString(unaryOp(op) + (operand match { + case Ast.expr.IntNum(_) | Ast.expr.FloatNum(_) => + translate(operand) + case _ => + s"(${translate(operand)})" + })) + case Ast.expr.IfExp(condition, ifTrue, ifFalse) => + trIfExp(condition, ifTrue, ifFalse) + case Ast.expr.Compare(left, op, right) => + (detectType(left), detectType(right)) match { + case (_: NumericType, _: NumericType) => + trNumericCompareOp(left, op, right) + case (_: StrType, _: StrType) => + trStrCompareOp(left, op, right) + case (_: BytesType, _: BytesType) => + trBytesCompareOp(left, op, right) + case (_: BooleanType, _: BooleanType) => + trNumericCompareOp(left, op, right) + case (_: EnumType, _: EnumType) => + trNumericCompareOp(left, op, right) + case (ltype, rtype) => + throw new TypeMismatchError(s"can't do $ltype $op $rtype") + } + case Ast.expr.EnumById(enumType, id, inType) => + val enumSpec = provider.resolveEnum(inType, enumType.name) + trEnumById(enumSpec.name, translate(id)) + case Ast.expr.EnumByLabel(enumType, label, inType) => + val enumSpec = provider.resolveEnum(inType, enumType.name) + trEnumByLabel(enumSpec.name, label.name) + case ctt: Ast.expr.CastToType => + doCastOrArray(ctt) + case Ast.expr.Subscript(container, idx) => + arraySubscript(container, idx) + case Ast.expr.Name(name: Ast.identifier) => + trLocalName(name.name) + case Ast.expr.List(elts) => + doGuessArrayLiteral(elts) + case call: Ast.expr.Attribute => + translateAttribute(call) + case call: Ast.expr.Call => + translateCall(call) + } + } + + def trIntLiteral(n: BigInt): TranslatorResult = ResultString(doIntLiteral(n)) + def trFloatLiteral(n: BigDecimal): TranslatorResult = ResultString(doFloatLiteral(n)) + def trStringLiteral(s: String): TranslatorResult = ResultString(doStringLiteral(s)) + def trBoolLiteral(n: Boolean): TranslatorResult = ResultString(doBoolLiteral(n)) + + def trBooleanOp(op: Ast.boolop, values: Seq[Ast.expr]) = + ResultString(doBooleanOp(op, values)) + + def trNumericBinOp(left: Ast.expr, op: Ast.operator, right: Ast.expr): TranslatorResult = { + (detectType(left), detectType(right), op) match { + case (t1: IntType, t2: IntType, Ast.operator.Mod) => + val v1 = allocateLocalVar() + out.puts(s"${localVarName(v1)} := ${translate(left)} % ${translate(right)}") + out.puts(s"if ${localVarName(v1)} < 0 {") + out.inc + out.puts(s"${localVarName(v1)} += ${translate(right)}") + out.dec + out.puts("}") + ResultLocalVar(v1) + case _ => + ResultString(numericBinOp(left, op, right)) + } + } + + def trStrConcat(left: Ast.expr, right: Ast.expr): TranslatorResult = + ResultString(translate(left) + " + " + translate(right)) + + def trNumericCompareOp(left: Ast.expr, op: Ast.cmpop, right: Ast.expr): TranslatorResult = + ResultString(doNumericCompareOp(left, op, right)) + + def trStrCompareOp(left: Ast.expr, op: Ast.cmpop, right: Ast.expr): TranslatorResult = + ResultString(doStrCompareOp(left, op, right)) + + def trBytesCompareOp(left: Ast.expr, op: Ast.cmpop, right: Ast.expr): TranslatorResult = { + importList.add("bytes") + op match { + case Ast.cmpop.Eq => + ResultString(s"bytes.Equal(${translate(left)}, ${translate(right)})") + case _ => + ResultString(s"(bytes.Compare(${translate(left)}, ${translate(right)}) ${cmpOp(op)} 0)") + } + } + + override def doIntLiteral(n: BigInt): String = { + if (n < -9223372036854775808L) { + s"$n" // too low, no type conversion would help anyway + } else if (n <= -2147483649L) { + s"int64($n)" // -9223372036854775808..–2147483649 + } else if (n <= 2147483647L) { + s"$n" // -2147483648..2147483647 + } else if (n <= 4294967295L) { + s"uint32($n)" // 2147483648..4294967295 + } else if (n <= 9223372036854775807L) { + s"int64($n)" // 4294967296..9223372036854775807 + } else if (n <= Utils.MAX_UINT64) { + s"uint64($n)" // 9223372036854775808..18446744073709551615 + } else { + s"$n" // too high, no type conversion would help anyway + } + } + + override def unaryOp(op: Ast.unaryop): String = op match { + case Ast.unaryop.Invert => "^" + case Ast.unaryop.Minus => "-" + case Ast.unaryop.Not => "!" + } + + def trLocalName(s: String): TranslatorResult = { + s match { + case Identifier.ROOT | + Identifier.PARENT | + Identifier.IO => + ResultString(s"this.${specialName(s)}") + + // These can be local only + case Identifier.ITERATOR | + Identifier.ITERATOR2 => + ResultString(specialName(s)) + + case _ => + if (provider.isLazy(s)) { + outVarCheckRes(s"this.${Utils.upperCamelCase(s)}()") + } else { + ResultString(s"this.${Utils.upperCamelCase(s)}") + } + } + } + + def specialName(id: String): String = id match { + case Identifier.ROOT | Identifier.PARENT | Identifier.IO => + id + case Identifier.ITERATOR => + "_it" + case Identifier.ITERATOR2 => + "_buf" + } + + def arraySubscript(container: Ast.expr, idx: Ast.expr) = + ResultString(s"${translate(container)}[${translate(idx)}]") + + def trIfExp(condition: Ast.expr, ifTrue: Ast.expr, ifFalse: Ast.expr): ResultLocalVar = { + val v1 = allocateLocalVar() + val typ = detectType(ifTrue) + out.puts(s"var ${localVarName(v1)} ${GoCompiler.kaitaiType2NativeType(typ)};") + out.puts(s"if (${translate(condition)}) {") + out.inc + out.puts(s"${localVarName(v1)} = ${translate(ifTrue)}") + out.dec + out.puts("} else {") + out.inc + out.puts(s"${localVarName(v1)} = ${translate(ifFalse)}") + out.dec + out.puts("}") + ResultLocalVar(v1) + } + + def trEnumByLabel(enumTypeAbs: List[String], label: String) = + ResultString(GoCompiler.enumToStr(enumTypeAbs, label)) + def trEnumById(enumTypeAbs: List[String], id: String) = + ResultString(s"${types2class(enumTypeAbs)}($id)") + + override def doBytesCompareOp(left: Ast.expr, op: Ast.cmpop, right: Ast.expr): String = { + op match { + case Ast.cmpop.Eq => + s"Arrays.equals(${translate(left)}, ${translate(right)})" + case Ast.cmpop.NotEq => + s"!Arrays.equals(${translate(left)}, ${translate(right)})" + case _ => + s"(${GoCompiler.kstreamName}.byteArrayCompare(${translate(left)}, ${translate(right)}) ${cmpOp(op)} 0)" + } + } + + override def doCast(value: Ast.expr, typeName: DataType): TranslatorResult = ??? + + override def doArrayLiteral(t: DataType, value: Seq[Ast.expr]) = + ResultString(s"[]${GoCompiler.kaitaiType2NativeType(t)}{${value.map(translate).mkString(", ")}}") + + override def doByteArrayLiteral(arr: Seq[Byte]): TranslatorResult = + ResultString("[]uint8{" + arr.map(_ & 0xff).mkString(", ") + "}") + + override def doByteArrayNonLiteral(elts: Seq[Ast.expr]): TranslatorResult = + ResultString("[]uint8{" + elts.map(translate).mkString(", ") + "}") + + // Predefined methods of various types + + val IMPORT_CHARMAP = "golang.org/x/text/encoding/charmap" + + val ENCODINGS = Map( + "cp437" -> ("charmap.CodePage437", IMPORT_CHARMAP), + "iso8859-1" -> ("charmap.ISO8859_1", IMPORT_CHARMAP), + "iso8859-2" -> ("charmap.ISO8859_2", IMPORT_CHARMAP), + "iso8859-3" -> ("charmap.ISO8859_3", IMPORT_CHARMAP), + "iso8859-4" -> ("charmap.ISO8859_4", IMPORT_CHARMAP), + "sjis" -> ("japanese.ShiftJIS", "golang.org/x/text/encoding/japanese"), + "big5" -> ("traditionalchinese.Big5", "golang.org/x/text/encoding/traditionalchinese") + ) + + override def bytesToStr(value: Ast.expr, expr: Ast.expr): TranslatorResult = + bytesToStr(translate(value), expr) + + def bytesToStr(bytesExpr: String, encoding: Ast.expr): TranslatorResult = { + val enc = encoding match { + case Ast.expr.Str(s) => s + case _ => throw new RuntimeException("Variable encodings are not supported in Go yet") + } + + enc.toLowerCase match { + case "ascii" | "utf-8" | "utf8" => + // no conversion + // FIXME: may be add some checks for valid ASCII/UTF-8 + ResultString(s"string($bytesExpr)") + case encStr => + ENCODINGS.get(encStr) match { + case Some((decoderSrc, importName)) => + importList.add(importName) + outVarCheckRes(s"kaitai.BytesToStr($bytesExpr, $decoderSrc.NewDecoder())") + case None => + throw new RuntimeException(s"encoding '$encStr' in not supported in Go") + } + } + } + +// override def strReverse(s: Ast.expr): String = +// s"new StringBuilder(${translate(s)}).reverse().toString()" +// override def strSubstring(s: Ast.expr, from: Ast.expr, to: Ast.expr): String = +// s"${translate(s)}.substring(${translate(from)}, ${translate(to)})" + + override def arrayFirst(a: Ast.expr): TranslatorResult = + ResultString(s"${translate(a)}[0]") + override def arrayLast(a: Ast.expr): ResultString = { + val v = allocateLocalVar() + out.puts(s"${localVarName(v)} := ${translate(a)}") + ResultString(s"${localVarName(v)}[len(${localVarName(v)}) - 1]") + } + override def arraySize(a: Ast.expr): TranslatorResult = + ResultString(s"len(${translate(a)})") +// override def arrayMin(a: Ast.expr): String = +// s"Collections.min(${translate(a)})" +// override def arrayMax(a: Ast.expr): String = +// s"Collections.max(${translate(a)})" + + override def userTypeField(ut: UserType, value: Ast.expr, name: String): TranslatorResult = { + val valueStr = translate(value) + + val (call, twoOuts) = name match { + case Identifier.ROOT | + Identifier.PARENT | + Identifier.IO => + (specialName(name), false) + case _ => + (Utils.upperCamelCase(name), provider.isLazy(ut.classSpec.get, name)) + } + + if (twoOuts) { + outVarCheckRes(s"$valueStr.$call()") + } else { + ResultString(s"$valueStr.$call") + } + } + + override def strLength(s: Ast.expr): TranslatorResult = { + importList.add("unicode/utf8") + ResultString(s"utf8.RuneCountInString(${translate(s)})") + } + + override def strReverse(s: Ast.expr): TranslatorResult = { + ResultString(s"kaitai.StringReverse(${translate(s)})") + } + + override def strToInt(s: Ast.expr, base: Ast.expr): TranslatorResult = { + importList.add("strconv") + ResultString(s"strconv.ParseInt(${translate(s)}, ${translate(base)}, 0)") + } + + override def strSubstring(s: Ast.expr, from: Ast.expr, to: Ast.expr): TranslatorResult = { + ResultString(s"${translate(s)}[${translate(from)}:${translate(to)}]") + } + + override def strToBytes(s: Ast.expr, encoding: Ast.expr): TranslatorResult = { + ResultString("") // TODO: implement + } + + override def intToStr(value: Ast.expr, base: Ast.expr): TranslatorResult = { + importList.add("strconv") + ResultString(s"strconv.FormatInt(int64(${translate(value)}), ${translate(base)})") + } + + override def floatToInt(value: Ast.expr) = + ResultString(s"int(${translate(value)})") + + override def kaitaiStreamSize(value: Ast.expr) = + outVarCheckRes(s"${translate(value)}.Size()") + + override def kaitaiStreamEof(value: Ast.expr) = + outVarCheckRes(s"${translate(value)}.EOF()") + + override def kaitaiStreamPos(value: Ast.expr) = + outVarCheckRes(s"${translate(value)}.Pos()") + + override def arrayMin(a: Ast.expr): ResultLocalVar = { + val min = allocateLocalVar() + val value = allocateLocalVar() + out.puts(s"${localVarName(min)} := ${translate(a)}[0]") + out.puts(s"for _, ${localVarName(value)} := range ${translate(a)} {") + out.inc + out.puts(s"if ${localVarName(min)} > ${localVarName(value)} {") + out.inc + out.puts(s"${localVarName(min)} = ${localVarName(value)}") + out.dec + out.puts("}") + out.dec + out.puts("}") + ResultLocalVar(min) + } + + override def arrayMax(a: Ast.expr): ResultLocalVar = { + val max = allocateLocalVar() + val value = allocateLocalVar() + out.puts(s"${localVarName(max)} := ${translate(a)}[0]") + out.puts(s"for _, ${localVarName(value)} := range ${translate(a)} {") + out.inc + out.puts(s"if ${localVarName(max)} < ${localVarName(value)} {") + out.inc + out.puts(s"${localVarName(max)} = ${localVarName(value)}") + out.dec + out.puts("}") + out.dec + out.puts("}") + ResultLocalVar(max) + } + + override def enumToInt(value: Ast.expr, et: EnumType) = + translateExpr(value) + + override def boolToInt(value: Ast.expr): ResultLocalVar = { + val v = allocateLocalVar() + out.puts(s"${localVarName(v)} := 0") + out.puts(s"if ${translate(value)} {") + out.inc + out.puts(s"${localVarName(v)} = 1") + out.dec + out.puts("}") + ResultLocalVar(v) + } + + def userType(dataType: UserType, io: String) = { + val v = allocateLocalVar() + out.puts(s"${localVarName(v)} := new(${GoCompiler.types2class(dataType.classSpec.get.name)})") + out.puts(s"err = ${localVarName(v)}.Read($io, this, this._root)") + outAddErrCheck() + ResultLocalVar(v) + } + + def outVarCheckRes(expr: String): ResultLocalVar = { + val v1 = allocateLocalVar() + out.puts(s"${localVarName(v1)}, err := $expr") + outAddErrCheck() + ResultLocalVar(v1) + } + + def outTransform(id: ResultLocalVar, expr: String): ResultLocalVar = { + out.puts(s"${resToStr(id)} = $expr") + id + } + + private + var localVarNum = 0 + + def allocateLocalVar(): Int = { + localVarNum += 1 + localVarNum + } + + def localVarName(n: Int) = s"tmp$n" + + def outAddErrCheck() { + out.puts("if err != nil {") + out.inc + + val noValueAndErr = returnRes match { + case None => "err" + case Some(r) => s"$r, err" + } + + out.puts(s"return $noValueAndErr") + out.dec + out.puts("}") + } + + override def byteSizeOfValue(attrName: String, valType: DataType): TranslatorResult = + trIntLiteral(CommonSizeOf.bitToByteSize(CommonSizeOf.getBitsSizeOfType(attrName, valType))) +} diff --git a/shared/src/main/scala/io/kaitai/struct/translators/JavaScriptTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/JavaScriptTranslator.scala index 714561e28..912ebfe68 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/JavaScriptTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/JavaScriptTranslator.scala @@ -4,15 +4,34 @@ import io.kaitai.struct.Utils import io.kaitai.struct.datatype.DataType._ import io.kaitai.struct.exprlang.Ast import io.kaitai.struct.exprlang.Ast.expr +import io.kaitai.struct.format.Identifier import io.kaitai.struct.languages.JavaScriptCompiler class JavaScriptTranslator(provider: TypeProvider) extends BaseTranslator(provider) { + override def doByteArrayNonLiteral(elts: Seq[Ast.expr]): String = + s"new Uint8Array([${elts.map(translate).mkString(", ")}])" + + /** + * JavaScript rendition of common control character that would use hex form, + * not octal. "Octal" control character string literals might be accepted + * in non-strict JS mode, but in strict mode only hex or unicode are ok. + * Here we'll use hex, as they are shorter. + * + * @see https://github.com/kaitai-io/kaitai_struct/issues/279 + * @param code character code to represent + * @return string literal representation of given code + */ + override def strLiteralGenericCC(code: Char): String = + "\\x%02x".format(code.toInt) + override def numericBinOp(left: Ast.expr, op: Ast.operator, right: Ast.expr) = { (detectType(left), detectType(right), op) match { case (_: IntType, _: IntType, Ast.operator.Div) => s"Math.floor(${translate(left)} / ${translate(right)})" case (_: IntType, _: IntType, Ast.operator.Mod) => s"${JavaScriptCompiler.kstreamName}.mod(${translate(left)}, ${translate(right)})" + case (_: IntType, _: IntType, Ast.operator.RShift) => + s"(${translate(left)} >>> ${translate(right)})" case _ => super.numericBinOp(left, op, right) } @@ -21,6 +40,8 @@ class JavaScriptTranslator(provider: TypeProvider) extends BaseTranslator(provid override def doLocalName(s: String) = { s match { case "_" => s + case Identifier.SWITCH_ON => "on" + case Identifier.INDEX => "i" case _ => s"this.${doName(s)}" } } @@ -41,7 +62,7 @@ class JavaScriptTranslator(provider: TypeProvider) extends BaseTranslator(provid override def doBytesCompareOp(left: Ast.expr, op: Ast.cmpop, right: Ast.expr): String = s"(${JavaScriptCompiler.kstreamName}.byteArrayCompare(${translate(left)}, ${translate(right)}) ${cmpOp(op)} 0)" - override def doSubscript(container: expr, idx: expr): String = + override def arraySubscript(container: expr, idx: expr): String = s"${translate(container)}[${translate(idx)}]" override def doIfExp(condition: expr, ifTrue: expr, ifFalse: expr): String = s"(${translate(condition)} ? ${translate(ifTrue)} : ${translate(ifFalse)})" @@ -66,6 +87,20 @@ class JavaScriptTranslator(provider: TypeProvider) extends BaseTranslator(provid override def boolToInt(v: expr): String = s"(${translate(v)} | 0)" + /** + * Converts a float to an integer in JavaScript. There are many methods to + * do so, here we use the fastest one, but it requires ES6+. OTOH, it is + * relatively easy to add compatibility polyfill for non-supporting environments + * (see MDN page). + * + * @see http://stackoverflow.com/a/596503/487064 + * @see https://developer.mozilla.org/en/docs/Web/JavaScript/Reference/Global_Objects/Math/trunc + * @param v float expression to convert + * @return string rendition of conversion + */ + override def floatToInt(v: expr): String = + s"Math.trunc(${translate(v)})" + override def intToStr(i: expr, base: expr): String = s"(${translate(i)}).toString(${translate(base)})" @@ -82,6 +117,9 @@ class JavaScriptTranslator(provider: TypeProvider) extends BaseTranslator(provid override def strSubstring(s: expr, from: expr, to: expr): String = s"${translate(s)}.substring(${translate(from)}, ${translate(to)})" + override def strToBytes(s: expr, encoding: expr): String = + "" // TODO: implement + override def arrayFirst(a: expr): String = s"${translate(a)}[0]" override def arrayLast(a: expr): String = { diff --git a/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala index c7a8f72a7..3997b0a45 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala @@ -1,32 +1,38 @@ package io.kaitai.struct.translators -import io.kaitai.struct.{ClassTypeProvider, RuntimeConfig, Utils} -import io.kaitai.struct.datatype.DataType -import io.kaitai.struct.datatype.DataType._ +import io.kaitai.struct.{ImportList, Utils} import io.kaitai.struct.exprlang.Ast import io.kaitai.struct.exprlang.Ast._ +import io.kaitai.struct.datatype.DataType +import io.kaitai.struct.datatype.DataType._ import io.kaitai.struct.format.Identifier import io.kaitai.struct.languages.JavaCompiler -class JavaTranslator(provider: TypeProvider, config: RuntimeConfig) extends BaseTranslator(provider) { +class JavaTranslator(provider: TypeProvider, importList: ImportList) extends BaseTranslator(provider) { override def doIntLiteral(n: BigInt): String = { - val literal = n.toString + val literal = if (n > Long.MaxValue) { + "0x" + n.toString(16) + } else { + n.toString + } val suffix = if (n > Int.MaxValue) "L" else "" - s"${literal}${suffix}" + s"$literal$suffix" } override def doArrayLiteral(t: DataType, value: Seq[expr]): String = { - // FIXME - val compiler = new JavaCompiler(provider.asInstanceOf[ClassTypeProvider], config) - - val javaType = compiler.kaitaiType2JavaTypeBoxed(t) + val javaType = JavaCompiler.kaitaiType2JavaTypeBoxed(t) val commaStr = value.map((v) => translate(v)).mkString(", ") + + importList.add("java.util.ArrayList") + importList.add("java.util.Arrays") s"new ArrayList<$javaType>(Arrays.asList($commaStr))" } override def doByteArrayLiteral(arr: Seq[Byte]): String = s"new byte[] { ${arr.mkString(", ")} }" + override def doByteArrayNonLiteral(elts: Seq[expr]): String = + s"new byte[] { ${elts.map(translate).mkString(", ")} }" override def numericBinOp(left: Ast.expr, op: Ast.operator, right: Ast.expr) = { (detectType(left), detectType(right), op) match { @@ -44,7 +50,8 @@ class JavaTranslator(provider: TypeProvider, config: RuntimeConfig) extends Base case Identifier.IO => "_io()" case Identifier.ITERATOR => "_it" case Identifier.ITERATOR2 => "_buf" - case Identifier.ITERATOR_I => "_i" + case Identifier.SWITCH_ON => "on" + case Identifier.INDEX => "i" case _ => s"${Utils.lowerCamelCase(s)}()" } @@ -71,46 +78,58 @@ class JavaTranslator(provider: TypeProvider, config: RuntimeConfig) extends Base override def doBytesCompareOp(left: Ast.expr, op: Ast.cmpop, right: Ast.expr): String = { op match { case Ast.cmpop.Eq => + importList.add("java.util.Arrays") s"Arrays.equals(${translate(left)}, ${translate(right)})" case Ast.cmpop.NotEq => + importList.add("java.util.Arrays") s"!Arrays.equals(${translate(left)}, ${translate(right)})" case _ => s"(${JavaCompiler.kstreamName}.byteArrayCompare(${translate(left)}, ${translate(right)}) ${cmpOp(op)} 0)" } } - override def doSubscript(container: expr, idx: expr): String = + override def arraySubscript(container: expr, idx: expr): String = s"${translate(container)}.get((int) ${translate(idx)})" override def doIfExp(condition: expr, ifTrue: expr, ifFalse: expr): String = s"(${translate(condition)} ? ${translate(ifTrue)} : ${translate(ifFalse)})" - override def doCast(value: Ast.expr, typeName: String): String = - s"((${Utils.upperCamelCase(typeName)}) (${translate(value)}))" + override def doCast(value: Ast.expr, typeName: DataType): String = + s"((${JavaCompiler.kaitaiType2JavaType(typeName)}) (${translate(value)}))" // Predefined methods of various types override def strToInt(s: expr, base: expr): String = s"Long.parseLong(${translate(s)}, ${translate(base)})" override def enumToInt(v: expr, et: EnumType): String = s"${translate(v)}.id()" + override def floatToInt(v: expr): String = + s"(int) (${translate(v)} + 0)" override def intToStr(i: expr, base: expr): String = s"Long.toString(${translate(i)}, ${translate(base)})" - override def bytesToStr(bytesExpr: String, encoding: Ast.expr): String = + override def bytesToStr(bytesExpr: String, encoding: Ast.expr): String = { + importList.add("java.nio.charset.Charset") s"new String($bytesExpr, Charset.forName(${translate(encoding)}))" - override def bytesFirst(b: Ast.expr): String = - s"${translate(b)}[0]" - override def bytesLast(b: Ast.expr): String = { - val v = translate(b) - s"$v[$v.length - 1]" } - override def bytesSize(b: Ast.expr): String = + + override def bytesLength(b: Ast.expr): String = s"${translate(b)}.length" - override def strToBytes(strExpr: String, encoding: expr): String = - s"($strExpr).getBytes(Charset.forName(${translate(encoding)}))" + override def bytesSubscript(container: Ast.expr, idx: Ast.expr): String = + s"${translate(container)}[${translate(idx)}]" + override def bytesFirst(b: Ast.expr): String = + s"${translate(b)}[0]" + override def bytesLast(b: Ast.expr): String = + s"${translate(b)}[(${translate(b)}).length - 1]" + override def bytesMin(b: Ast.expr): String = + s"${JavaCompiler.kstreamName}.byteArrayMin(${translate(b)})" + override def bytesMax(b: Ast.expr): String = + s"${JavaCompiler.kstreamName}.byteArrayMax(${translate(b)})" + override def strLength(s: expr): String = - s"${translate(s)}.length()" + s"(${translate(s)}).length()" override def strReverse(s: expr): String = s"new StringBuilder(${translate(s)}).reverse().toString()" override def strSubstring(s: expr, from: expr, to: expr): String = - s"${translate(s)}.substring(${translate(from)}, ${translate(to)})" + s"(${translate(s)}).substring(${translate(from)}, ${translate(to)})" + override def strToBytes(s: expr, encoding: expr): String = + s"(${translate(s)}).getBytes(Charset.forName(${translate(encoding)}))" override def arrayFirst(a: expr): String = s"${translate(a)}.get(0)" @@ -120,8 +139,12 @@ class JavaTranslator(provider: TypeProvider, config: RuntimeConfig) extends Base } override def arraySize(a: expr): String = s"${translate(a)}.size()" - override def arrayMin(a: Ast.expr): String = + override def arrayMin(a: Ast.expr): String = { + importList.add("java.util.Collections") s"Collections.min(${translate(a)})" - override def arrayMax(a: Ast.expr): String = + } + override def arrayMax(a: Ast.expr): String = { + importList.add("java.util.Collections") s"Collections.max(${translate(a)})" + } } diff --git a/shared/src/main/scala/io/kaitai/struct/translators/LuaTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/LuaTranslator.scala new file mode 100644 index 000000000..e0be2091c --- /dev/null +++ b/shared/src/main/scala/io/kaitai/struct/translators/LuaTranslator.scala @@ -0,0 +1,159 @@ +package io.kaitai.struct.translators + +import io.kaitai.struct.ImportList +import io.kaitai.struct.datatype.DataType +import io.kaitai.struct.datatype.DataType._ +import io.kaitai.struct.format.Identifier +import io.kaitai.struct.exprlang.Ast +import io.kaitai.struct.languages.LuaCompiler + +class LuaTranslator(provider: TypeProvider, importList: ImportList) extends BaseTranslator(provider) { + override val asciiCharQuoteMap: Map[Char, String] = Map( + '\t' -> "\\t", + '\n' -> "\\n", + '\r' -> "\\r", + '"' -> "\\\"", + '\\' -> "\\\\", + + '\7' -> "\\a", + '\b' -> "\\b", + '\13' -> "\\v", + '\f' -> "\\f", + '\33' -> "\\027" + ) + + override def strLiteralUnicode(code: Char): String = + "\\u{%04x}".format(code.toInt) + + override def arraySubscript(container: Ast.expr, idx: Ast.expr): String = { + // Lua indexes start at 1, so we need to offset them + val fixedIdx = idx match { + case Ast.expr.IntNum(n) => Ast.expr.IntNum(n + 1) + case _ => idx + } + + s"${translate(container)}[${translate(fixedIdx)}]" + } + override def doIfExp(condition: Ast.expr, ifTrue: Ast.expr, ifFalse: Ast.expr): String = + s"(((${translate(condition)}) and (${translate(ifTrue)})) or (${translate(ifFalse)}))" + + override def doBoolLiteral(n: Boolean): String = + if (n) "true" else "false" + override def doArrayLiteral(t: DataType, value: Seq[Ast.expr]): String = + "{" + value.map((v) => translate(v)).mkString(", ") + "}" + override def doByteArrayLiteral(arr: Seq[Byte]): String = + "\"" + decEscapeByteArray(arr) + "\"" + + override def doLocalName(s: String) = s match { + case Identifier.ITERATOR => "_" + case Identifier.INDEX => "i" + case _ => s"self.${doName(s)}" + } + override def doName(s: String): String = + s + override def doEnumByLabel(enumTypeAbs: List[String], label: String): String = + s"${LuaCompiler.types2class(enumTypeAbs)}.$label" + override def doEnumById(enumTypeAbs: List[String], id: String): String = + s"${LuaCompiler.types2class(enumTypeAbs)}($id)" + + // This is very hacky because integers and booleans cannot be compared + override def doNumericCompareOp(left: Ast.expr, op: Ast.cmpop, right: Ast.expr): String = { + val bool2Int = (n: Boolean) => { if (n) "1" else "0" } + (left, right) match { + case (Ast.expr.Bool(l), Ast.expr.Bool(r)) => s"${bool2Int(l)} ${cmpOp(op)} ${bool2Int(r)}" + case (Ast.expr.Bool(l), r) => s"${bool2Int(l)} ${cmpOp(op)} ${translate(r)}" + case (l, Ast.expr.Bool(r)) => s"${translate(l)} ${cmpOp(op)} ${bool2Int(r)}" + case _ => super.doNumericCompareOp(left, op, right) + } + } + + override def strConcat(left: Ast.expr, right: Ast.expr): String = + s"${translate(left)} .. ${translate(right)}" + override def strToInt(s: Ast.expr, base: Ast.expr): String = { + val baseStr = translate(base) + val add = baseStr match { + case "10" => "" + case _ => s", $baseStr" + } + s"tonumber(${translate(s)}$add)" + } + override def enumToInt(v: Ast.expr, et: EnumType): String = + s"${translate(v)}.value" + override def boolToInt(v: Ast.expr): String = + s"(${translate(v)} and 1 or 0)" + override def floatToInt(v: Ast.expr): String = + s"(${translate(v)} > 0) and math.floor(${translate(v)}) or math.ceil(${translate(v)})" + override def intToStr(i: Ast.expr, base: Ast.expr): String = { + val baseStr = translate(base) + baseStr match { + case "10" => s"tostring(${translate(i)})" + case _ => throw new UnsupportedOperationException(baseStr) + } + } + override def bytesToStr(bytesExpr: String, encoding: Ast.expr): String = { + importList.add("local str_decode = require(\"string_decode\")") + + s"str_decode.decode($bytesExpr, ${translate(encoding)})" + } + override def strLength(s: Ast.expr): String = + s"string.len(${translate(s)})" + override def strReverse(s: Ast.expr): String = + s"string.reverse(${translate(s)})" + override def strSubstring(s: Ast.expr, from: Ast.expr, to: Ast.expr): String = + s"string.sub(${translate(s)}, ${translate(from)}, ${translate(to)})" + override def strToBytes(s: Ast.expr, encoding: Ast.expr): String = + "" // TODO: implement + + override def arrayFirst(a: Ast.expr): String = + s"${translate(a)}[1]" + override def arrayLast(a: Ast.expr): String = { + val table = translate(a) + s"${table}[#${table}]" + } + override def arraySize(a: Ast.expr): String = + s"#${translate(a)}" + override def arrayMin(a: Ast.expr): String = { + importList.add("local utils = require(\"utils\")") + + s"utils.array_min(${translate(a)})" + } + override def arrayMax(a: Ast.expr): String ={ + importList.add("local utils = require(\"utils\")") + + s"utils.array_max(${translate(a)})" + } + + override def kaitaiStreamSize(value: Ast.expr): String = + s"${translate(value)}:size()" + override def kaitaiStreamEof(value: Ast.expr): String = + s"${translate(value)}:is_eof()" + override def kaitaiStreamPos(value: Ast.expr): String = + s"${translate(value)}:pos()" + + override def binOp(op: Ast.operator): String = op match { + case Ast.operator.BitXor => "~" + case _ => super.binOp(op) + } + override def cmpOp(op: Ast.cmpop): String = op match { + case Ast.cmpop.NotEq => "~=" + case _ => super.cmpOp(op) + } + override def booleanOp(op: Ast.boolop): String = op match { + case Ast.boolop.Or => "or" + case Ast.boolop.And => "and" + } + override def unaryOp(op: Ast.unaryop): String = op match { + case Ast.unaryop.Not => "not" + case _ => super.unaryOp(op) + } + + /** + * Converts byte array (Seq[Byte]) into decimal-escaped Lua-style literal + * characters (i.e. like \255). + * + * @param arr byte array to escape + * @return array contents decimal-escaped as string + */ + private def decEscapeByteArray(arr: Seq[Byte]): String = + arr.map((x) => "\\%03d".format(x & 0xff)).mkString +} diff --git a/shared/src/main/scala/io/kaitai/struct/translators/NimTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/NimTranslator.scala new file mode 100644 index 000000000..d617754a7 --- /dev/null +++ b/shared/src/main/scala/io/kaitai/struct/translators/NimTranslator.scala @@ -0,0 +1,102 @@ +package io.kaitai.struct.translators + +import io.kaitai.struct.{ImportList, Utils} +import io.kaitai.struct.exprlang.Ast +import io.kaitai.struct.exprlang.Ast._ +import io.kaitai.struct.datatype.DataType +import io.kaitai.struct.datatype.DataType._ +import io.kaitai.struct.format.Identifier +import io.kaitai.struct.NimClassCompiler.ksToNim + +class NimTranslator(provider: TypeProvider, importList: ImportList) extends BaseTranslator(provider) { + // Members declared in io.kaitai.struct.translators.BaseTranslator + override def bytesToStr(bytesExpr: String, encoding: Ast.expr): String = { + importList.add("encodings") + s"convert($bytesExpr, srcEncoding = ${translate(encoding)})" + } + override def doEnumById(enumTypeAbs: List[String], id: String): String = "" + override def doEnumByLabel(enumTypeAbs: List[String], label: String): String = "" + override def doName(s: String): String = + s match { + case Identifier.PARENT => "parent" + case Identifier.IO => "stream" + case Identifier.ITERATOR2 => "it" + case _ => s"${Utils.lowerCamelCase(s)}" + } + override def doIfExp(condition: expr, ifTrue: expr, ifFalse: expr): String = + s"(if ${translate(condition)}: ${translate(ifTrue)} else: ${translate(ifFalse)})" + override def arraySubscript(container: expr, idx: expr): String = + s"${translate(container)}[${translate(idx)}]" + + override def strConcat(left: Ast.expr, right: Ast.expr): String = s"${translate(left)} & ${translate(right)}" + + // Members declared in io.kaitai.struct.translators.CommonMethods + + override def unaryOp(op: Ast.unaryop): String = op match { + case Ast.unaryop.Invert => "not" + case Ast.unaryop.Minus => "-" + case Ast.unaryop.Not => "not" + } + + override def binOp(op: Ast.operator): String = { + op match { + case Ast.operator.Add => "+" + case Ast.operator.Sub => "-" + case Ast.operator.Mult => "*" + case Ast.operator.Div => "/" + case Ast.operator.Mod => "%%%" + case Ast.operator.BitAnd => "and" + case Ast.operator.BitOr => "or" + case Ast.operator.BitXor => "xor" + case Ast.operator.LShift => "shl" + case Ast.operator.RShift => "shr" + } + } +// override def doCast(value: Ast.expr, typeName: DataType): String = +// s"${NimCompiler.kaitaiType2NimType(typeName)}(${translate(value)})" + override def doIntLiteral(n: BigInt): String = { + if (n < -9223372036854775808L) { + s"$n" // too low, no type conversion would help anyway + } else if (n <= -2147483649L) { + s"$n'i64" // -9223372036854775808..–2147483649 + } else if (n <= 2147483647L) { + s"$n" // -2147483648..2147483647 + } else if (n <= 4294967295L) { + s"$n'u32" // 2147483648..4294967295 + } else if (n <= 9223372036854775807L) { + s"$n'i64" // 4294967296..9223372036854775807 + } else if (n <= Utils.MAX_UINT64) { + s"$n'u64" // 9223372036854775808..18446744073709551615 + } else { + s"$n" // too high, no type conversion would help anyway + } + } + override def doArrayLiteral(t: DataType, value: Seq[expr]): String = + s"@[${value.map((v) => translate(v)).mkString(", ")}].mapIt(${ksToNim(t)}(it))" + override def doByteArrayLiteral(arr: Seq[Byte]): String = + s"@[${arr.mkString(", ")}].mapIt(toByte(it))" + override def doByteArrayNonLiteral(elts: Seq[expr]): String = + s"@[${elts.map(translate).mkString(", ")}]" + override def arrayFirst(a: expr): String = s"${translate(a)}[0]" + override def arrayLast(a: expr): String = s"${translate(a)}[^1]" + override def arrayMax(a: expr): String = s"max(${translate(a)})" + override def arrayMin(a: expr): String = s"min(${translate(a)})" + override def arraySize(a: expr): String = s"len(${translate(a)})" + override def enumToInt(v: expr, et: EnumType): String = s"ord(${translate(v)})" + override def floatToInt(v: expr): String = s"int(${translate(v)})" + override def intToStr(v: expr, base: expr): String = { + importList.add("strutils") + s"intToStr(${translate(v)}.parseInt(${translate(base)}))" + } + override def strLength(s: expr): String = s"len(${translate(s)})" + override def strReverse(s: expr): String = { + importList.add("unicode") + s"reversed(${translate(s)})" + } + override def strSubstring(s: expr, from: expr, to: expr): String = + s"${translate(s)}.substr(${translate(from)}, ${translate(to)})" + override def strToBytes(s: expr, encoding: expr): String = + "" // TODO: implement + override def strToInt(s: expr, base: expr): String = + s"${translate(s)}.parseInt(${translate(base)}" +} diff --git a/shared/src/main/scala/io/kaitai/struct/translators/PHPTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/PHPTranslator.scala index e0c3f6632..2cc8f6742 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/PHPTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/PHPTranslator.scala @@ -10,6 +10,8 @@ import io.kaitai.struct.{RuntimeConfig, Utils} class PHPTranslator(provider: TypeProvider, config: RuntimeConfig) extends BaseTranslator(provider) { override def doByteArrayLiteral(arr: Seq[Byte]): String = "\"" + Utils.hexEscapeByteArray(arr) + "\"" + override def doByteArrayNonLiteral(elts: Seq[Ast.expr]): String = + s"pack('C*', ${elts.map(translate).mkString(", ")})" // http://php.net/manual/en/language.types.string.php#language.types.string.syntax.double override val asciiCharQuoteMap: Map[Char, String] = Map( @@ -41,13 +43,14 @@ class PHPTranslator(provider: TypeProvider, config: RuntimeConfig) extends BaseT } } - override def userTypeField(value: expr, attrName: String): String = + override def anyField(value: expr, attrName: String): String = s"${translate(value)}->${doName(attrName)}" override def doLocalName(s: String) = { s match { case Identifier.ITERATOR => "$_" case Identifier.ITERATOR2 => "$_buf" + case Identifier.INDEX => "$i" case _ => s"$$this->${doName(s)}" } } @@ -62,7 +65,7 @@ class PHPTranslator(provider: TypeProvider, config: RuntimeConfig) extends BaseT // Just an integer, without any casts / resolutions - one would have to look up constants manually id - override def doSubscript(container: expr, idx: expr): String = + override def arraySubscript(container: expr, idx: expr): String = s"${translate(container)}[${translate(idx)}]" override def doIfExp(condition: expr, ifTrue: expr, ifFalse: expr): String = s"(${translate(condition)} ? ${translate(ifTrue)} : ${translate(ifFalse)})" @@ -80,6 +83,9 @@ class PHPTranslator(provider: TypeProvider, config: RuntimeConfig) extends BaseT override def boolToInt(v: expr): String = s"intval(${translate(v)})" + override def floatToInt(v: expr): String = + s"intval(${translate(v)})" + override def intToStr(i: expr, base: expr): String = { val baseStr = translate(base) baseStr match { @@ -91,12 +97,16 @@ class PHPTranslator(provider: TypeProvider, config: RuntimeConfig) extends BaseT } override def bytesToStr(bytesExpr: String, encoding: Ast.expr): String = s"${PHPCompiler.kstreamName}::bytesToStr($bytesExpr, ${translate(encoding)})" + override def bytesLength(b: Ast.expr): String = + s"strlen(${translate(b)})" override def strLength(s: expr): String = s"strlen(${translate(s)})" override def strReverse(s: expr): String = s"strrev(${translate(s)})" override def strSubstring(s: expr, from: expr, to: expr): String = s"${translate(s)}.substring(${translate(from)}, ${translate(to)})" + override def strToBytes(s: expr, encoding: expr): String = + "" // TODO: implement override def arrayFirst(a: expr): String = s"${translate(a)}[0]" diff --git a/shared/src/main/scala/io/kaitai/struct/translators/PerlTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/PerlTranslator.scala index 381981af9..52633b7cc 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/PerlTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/PerlTranslator.scala @@ -1,11 +1,13 @@ package io.kaitai.struct.translators +import io.kaitai.struct.{ImportList, Utils} import io.kaitai.struct.datatype.DataType import io.kaitai.struct.datatype.DataType._ import io.kaitai.struct.exprlang.Ast import io.kaitai.struct.format.Identifier +import io.kaitai.struct.languages.PerlCompiler -class PerlTranslator(provider: TypeProvider) extends BaseTranslator(provider) { +class PerlTranslator(provider: TypeProvider, importList: ImportList) extends BaseTranslator(provider) { // http://perldoc.perl.org/perlrebackslash.html#Character-Escapes override val asciiCharQuoteMap: Map[Char, String] = Map( '\t' -> "\\t", @@ -48,13 +50,16 @@ class PerlTranslator(provider: TypeProvider) extends BaseTranslator(provider) { override def doByteArrayLiteral(arr: Seq[Byte]): String = s"pack('C*', (${arr.map(_ & 0xff).mkString(", ")}))" + override def doByteArrayNonLiteral(elts: Seq[Ast.expr]): String = + s"pack('C*', (${elts.map(translate).mkString(", ")}))" - override def userTypeField(value: Ast.expr, attrName: String): String = + override def anyField(value: Ast.expr, attrName: String): String = s"${translate(value)}->${doName(attrName)}" override def doLocalName(s: String) = { s match { case "_" | "_on" => "$" + s + case Identifier.INDEX => doName(s) case _ => s"$$self->${doName(s)}" } } @@ -63,16 +68,26 @@ class PerlTranslator(provider: TypeProvider) extends BaseTranslator(provider) { s match { case Identifier.ITERATOR => "$_" case Identifier.ITERATOR2 => "$_buf" + case Identifier.INDEX => "$i" case _ => s"$s()" } } - override def doEnumByLabel(enumType: List[String], label: String): String = - s"$$${enumType.last.toUpperCase}_${label.toUpperCase}" + override def doEnumByLabel(enumType: List[String], label: String): String = { + val enumClass = PerlCompiler.types2class(enumType.init) + val enumClassWithScope = if (enumClass.isEmpty) "" else s"$enumClass::" + val enumName = enumType.last.toUpperCase + s"$$$enumClassWithScope${enumName}_${label.toUpperCase}" + } override def doEnumById(enumTypeAbs: List[String], id: String): String = // Just an integer, without any casts / resolutions - one would have to look up constants manually id + def enumClass(enumTypeAbs: List[String]): String = { + val enumTypeRel = Utils.relClass(enumTypeAbs, provider.nowClass.name) + enumTypeRel.map((x) => Utils.upperCamelCase(x)).mkString(".") + } + override def doStrCompareOp(left: Ast.expr, op: Ast.cmpop, right: Ast.expr) = { val opStr = op match { case Ast.cmpop.Eq => "eq" @@ -88,8 +103,8 @@ class PerlTranslator(provider: TypeProvider) extends BaseTranslator(provider) { override def doBytesCompareOp(left: Ast.expr, op: Ast.cmpop, right: Ast.expr): String = doStrCompareOp(left, op, right) - override def doSubscript(container: Ast.expr, idx: Ast.expr): String = - s"${translate(container)}[${translate(idx)}]" + override def arraySubscript(container: Ast.expr, idx: Ast.expr): String = + s"@{${translate(container)}}[${translate(idx)}]" override def doIfExp(condition: Ast.expr, ifTrue: Ast.expr, ifFalse: Ast.expr): String = s"(${translate(condition)} ? ${translate(ifTrue)} : ${translate(ifFalse)})" @@ -112,6 +127,8 @@ class PerlTranslator(provider: TypeProvider) extends BaseTranslator(provider) { translate(v) override def boolToInt(v: Ast.expr): String = translate(v) + override def floatToInt(v: Ast.expr): String = + s"int(${translate(v)})" override def intToStr(i: Ast.expr, base: Ast.expr): String = { val baseStr = translate(base) val format = baseStr match { @@ -126,14 +143,18 @@ class PerlTranslator(provider: TypeProvider) extends BaseTranslator(provider) { s"sprintf('$format', ${translate(i)})" } - override def bytesToStr(bytesExpr: String, encoding: Ast.expr): String = + override def bytesToStr(bytesExpr: String, encoding: Ast.expr): String = { + importList.add("Encode") s"Encode::decode(${translate(encoding)}, $bytesExpr)" + } override def strLength(value: Ast.expr): String = s"length(${translate(value)})" override def strReverse(value: Ast.expr): String = s"scalar(reverse(${translate(value)}))" override def strSubstring(s: Ast.expr, from: Ast.expr, to: Ast.expr): String = s"${translate(s)}[${translate(from)}:${translate(to)}]" + override def strToBytes(s: Ast.expr, encoding: Ast.expr): String = + "" // TODO: implement override def arrayFirst(a: Ast.expr): String = s"@{${translate(a)}}[0]" @@ -146,6 +167,7 @@ class PerlTranslator(provider: TypeProvider) extends BaseTranslator(provider) { case _: StrType => "minstr" case _ => "min" } + importList.add("List::Util") s"List::Util::$funcName(@{${translate(a)}})" } override def arrayMax(a: Ast.expr): String = { @@ -153,6 +175,7 @@ class PerlTranslator(provider: TypeProvider) extends BaseTranslator(provider) { case _: StrType => "maxstr" case _ => "max" } + importList.add("List::Util") s"List::Util::$funcName(@{${translate(a)}})" } diff --git a/shared/src/main/scala/io/kaitai/struct/translators/PythonTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/PythonTranslator.scala index bd190ffec..1e5a65b2b 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/PythonTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/PythonTranslator.scala @@ -1,10 +1,12 @@ package io.kaitai.struct.translators +import io.kaitai.struct.{ImportList, Utils} import io.kaitai.struct.datatype.DataType._ import io.kaitai.struct.exprlang.Ast -import io.kaitai.struct.languages.PythonCompiler +import io.kaitai.struct.format.Identifier +import io.kaitai.struct.languages.{PythonCompiler, RubyCompiler} -class PythonTranslator(provider: TypeProvider) extends BaseTranslator(provider) { +class PythonTranslator(provider: TypeProvider, importList: ImportList) extends BaseTranslator(provider) { override def numericBinOp(left: Ast.expr, op: Ast.operator, right: Ast.expr) = { (detectType(left), detectType(right), op) match { case (_: IntType, _: IntType, Ast.operator.Div) => @@ -35,11 +37,16 @@ class PythonTranslator(provider: TypeProvider) extends BaseTranslator(provider) ) override def doByteArrayLiteral(arr: Seq[Byte]): String = - s"struct.pack('${arr.length}b', ${arr.mkString(", ")})" + "b\"" + Utils.hexEscapeByteArray(arr) + "\"" + override def doByteArrayNonLiteral(elts: Seq[Ast.expr]): String = { + importList.add("import struct") + s"struct.pack('${elts.length}b', ${elts.map(translate).mkString(", ")})" + } override def doLocalName(s: String) = { s match { - case "_" => s + case Identifier.ITERATOR => "_" + case Identifier.INDEX => "i" case _ => s"self.${doName(s)}" } } @@ -47,8 +54,8 @@ class PythonTranslator(provider: TypeProvider) extends BaseTranslator(provider) override def doEnumByLabel(enumTypeAbs: List[String], label: String): String = s"${PythonCompiler.types2class(enumTypeAbs)}.$label" - override def doEnumById(enumTypeAbs: List[String], id: String) = - s"${PythonCompiler.types2class(enumTypeAbs)}($id)" + override def doEnumById(enumTypeAbs: List[String], id: String): String = + s"${PythonCompiler.kstreamName}.resolve_enum(${PythonCompiler.types2class(enumTypeAbs)}, $id)" override def booleanOp(op: Ast.boolop) = op match { case Ast.boolop.Or => "or" @@ -60,7 +67,7 @@ class PythonTranslator(provider: TypeProvider) extends BaseTranslator(provider) case _ => super.unaryOp(op) } - override def doSubscript(container: Ast.expr, idx: Ast.expr): String = + override def arraySubscript(container: Ast.expr, idx: Ast.expr): String = s"${translate(container)}[${translate(idx)}]" override def doIfExp(condition: Ast.expr, ifTrue: Ast.expr, ifFalse: Ast.expr): String = s"(${translate(ifTrue)} if ${translate(condition)} else ${translate(ifFalse)})" @@ -78,6 +85,8 @@ class PythonTranslator(provider: TypeProvider) extends BaseTranslator(provider) s"${translate(v)}.value" override def boolToInt(v: Ast.expr): String = s"int(${translate(v)})" + override def floatToInt(v: Ast.expr): String = + s"int(${translate(v)})" override def intToStr(i: Ast.expr, base: Ast.expr): String = { val baseStr = translate(base) val func = baseStr match { @@ -92,12 +101,16 @@ class PythonTranslator(provider: TypeProvider) extends BaseTranslator(provider) } override def bytesToStr(bytesExpr: String, encoding: Ast.expr): String = s"($bytesExpr).decode(${translate(encoding)})" + override def bytesLength(value: Ast.expr): String = + s"len(${translate(value)})" override def strLength(value: Ast.expr): String = s"len(${translate(value)})" override def strReverse(value: Ast.expr): String = s"${translate(value)}[::-1]" override def strSubstring(s: Ast.expr, from: Ast.expr, to: Ast.expr): String = s"${translate(s)}[${translate(from)}:${translate(to)}]" + override def strToBytes(s: Ast.expr, encoding: Ast.expr): String = + "" // TODO: implement override def arrayFirst(a: Ast.expr): String = s"${translate(a)}[0]" diff --git a/shared/src/main/scala/io/kaitai/struct/translators/RubyTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/RubyTranslator.scala index 3fc8aa870..ee724115a 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/RubyTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/RubyTranslator.scala @@ -1,13 +1,17 @@ package io.kaitai.struct.translators +import io.kaitai.struct.Utils import io.kaitai.struct.datatype.DataType.EnumType import io.kaitai.struct.exprlang.Ast -import io.kaitai.struct.exprlang.Ast.expr +import io.kaitai.struct.format.Identifier import io.kaitai.struct.languages.RubyCompiler -class RubyTranslator(provider: TypeProvider) extends BaseTranslator(provider) { +class RubyTranslator(provider: TypeProvider) extends BaseTranslator(provider) + with ByteArraysAsTrueArrays[String] { override def doByteArrayLiteral(arr: Seq[Byte]): String = s"${super.doByteArrayLiteral(arr)}.pack('C*')" + override def doByteArrayNonLiteral(elts: Seq[Ast.expr]): String = + s"[${elts.map(translate).mkString(", ")}].pack('C*')" // https://github.com/ruby/ruby/blob/trunk/doc/syntax/literals.rdoc#strings // https://github.com/ruby/ruby/blob/trunk/string.c - see "rb_str_inspect" @@ -26,14 +30,32 @@ class RubyTranslator(provider: TypeProvider) extends BaseTranslator(provider) { '\b' -> "\\b" ) - override def doName(s: String) = s + override def doName(s: String) = { + s match { + case Identifier.INDEX => "i" // FIXME: probably would clash with attribute named "i" + case _ => s + } + } override def doEnumByLabel(enumTypeAbs: List[String], label: String): String = s":${enumTypeAbs.last}_$label" override def doEnumById(enumType: List[String], id: String): String = - s"${RubyCompiler.kstreamName}::resolve_enum(${enumType.last.toUpperCase}, $id)" + s"${RubyCompiler.kstreamName}::resolve_enum(${enumDirectMap(enumType)}, $id)" + + def enumDirectMap(enumTypeAndName: List[String]): String = { + val enumTypeAbs = enumTypeAndName.dropRight(1) + val enumTypeName = enumTypeAndName.last.toUpperCase - override def doSubscript(container: Ast.expr, idx: Ast.expr): String = + val enumTypeRel = Utils.relClass(enumTypeAbs, provider.nowClass.name) + + if (enumTypeRel.nonEmpty) { + (enumTypeRel.map((x) => Utils.upperCamelCase(x)) ++ List(enumTypeName)).mkString("::") + } else { + enumTypeName + } + } + + override def arraySubscript(container: Ast.expr, idx: Ast.expr): String = s"${translate(container)}[${translate(idx)}]" override def doIfExp(condition: Ast.expr, ifTrue: Ast.expr, ifFalse: Ast.expr): String = s"(${translate(condition)} ? ${translate(ifTrue)} : ${translate(ifFalse)})" @@ -48,16 +70,40 @@ class RubyTranslator(provider: TypeProvider) extends BaseTranslator(provider) { } override def enumToInt(v: Ast.expr, et: EnumType): String = s"${RubyCompiler.inverseEnumName(et.name.last.toUpperCase)}[${translate(v)}]" + override def floatToInt(v: Ast.expr): String = + s"(${translate(v)}).to_i" override def intToStr(i: Ast.expr, base: Ast.expr): String = translate(i) + s".to_s(${translate(base)})" + override def bytesToStr(bytesExpr: String, encoding: Ast.expr): String = s"($bytesExpr).force_encoding(${translate(encoding)})" + override def bytesLength(b: Ast.expr): String = + s"${translate(b)}.size" + /** + * Alternatives considered: + * + * * value[0].ord => 6341 => winner by performance + * * value.bytes[0] => 8303 + */ + override def bytesSubscript(container: Ast.expr, idx: Ast.expr): String = + s"${translate(container)}[${translate(idx)}].ord" + override def bytesFirst(b: Ast.expr): String = + s"${translate(b)}[0].ord" + override def bytesLast(b: Ast.expr): String = + s"${translate(b)}[-1].ord" + override def bytesMin(b: Ast.expr): String = + s"${translate(b)}.bytes.min" + override def bytesMax(b: Ast.expr): String = + s"${translate(b)}.bytes.max" + override def strLength(s: Ast.expr): String = s"${translate(s)}.size" override def strReverse(s: Ast.expr): String = s"${translate(s)}.reverse" override def strSubstring(s: Ast.expr, from: Ast.expr, to: Ast.expr): String = s"${translate(s)}[${translate(from)}, (${translate(to)} - 1)]" + override def strToBytes(s: Ast.expr, encoding: Ast.expr): String = + "" // TODO: implement override def arrayFirst(a: Ast.expr): String = s"${translate(a)}.first" @@ -65,9 +111,9 @@ class RubyTranslator(provider: TypeProvider) extends BaseTranslator(provider) { s"${translate(a)}.last" override def arraySize(a: Ast.expr): String = s"${translate(a)}.length" - override def arrayMin(a: expr): String = + override def arrayMin(a: Ast.expr): String = s"${translate(a)}.min" - override def arrayMax(a: expr): String = + override def arrayMax(a: Ast.expr): String = s"${translate(a)}.max" override def kaitaiStreamEof(value: Ast.expr): String = diff --git a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala new file mode 100644 index 000000000..4a12e77a5 --- /dev/null +++ b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala @@ -0,0 +1,131 @@ +package io.kaitai.struct.translators + +import io.kaitai.struct.datatype.DataType._ +import io.kaitai.struct.exprlang.Ast +import io.kaitai.struct.exprlang.Ast.expr +import io.kaitai.struct.format.Identifier +import io.kaitai.struct.languages.RustCompiler +import io.kaitai.struct.{RuntimeConfig, Utils} + +class RustTranslator(provider: TypeProvider, config: RuntimeConfig) extends BaseTranslator(provider) { + override def doByteArrayLiteral(arr: Seq[Byte]): String = + "vec!([" + arr.map((x) => + "%0#2x".format(x & 0xff) + ).mkString(", ") + "])" + override def doByteArrayNonLiteral(elts: Seq[Ast.expr]): String = + s"pack('C*', ${elts.map(translate).mkString(", ")})" + + override val asciiCharQuoteMap: Map[Char, String] = Map( + '\t' -> "\\t", + '\n' -> "\\n", + '\r' -> "\\r", + '"' -> "\\\"", + '\\' -> "\\\\" + ) + + override def strLiteralUnicode(code: Char): String = + "\\u{%x}".format(code.toInt) + + override def numericBinOp(left: Ast.expr, op: Ast.operator, right: Ast.expr) = { + (detectType(left), detectType(right), op) match { + case (_: IntType, _: IntType, Ast.operator.Div) => + s"${translate(left)} / ${translate(right)}" + case (_: IntType, _: IntType, Ast.operator.Mod) => + s"${translate(left)} % ${translate(right)}" + case _ => + super.numericBinOp(left, op, right) + } + } + + override def doLocalName(s: String) = { + s match { + case Identifier.ITERATOR => "tmpa" + case Identifier.ITERATOR2 => "tmpb" + case Identifier.INDEX => "i" + case _ => s"self.${doName(s)}" + } + } + + override def doName(s: String) = s + + override def doEnumByLabel(enumTypeAbs: List[String], label: String): String = { + val enumClass = types2classAbs(enumTypeAbs) + s"$enumClass::${label.toUpperCase}" + } + override def doEnumById(enumTypeAbs: List[String], id: String) = + // Just an integer, without any casts / resolutions - one would have to look up constants manually + id + + override def arraySubscript(container: expr, idx: expr): String = + s"${translate(container)}[${translate(idx)}]" + override def doIfExp(condition: expr, ifTrue: expr, ifFalse: expr): String = + "if " + translate(condition) + + " { " + translate(ifTrue) + " } else { " + + translate(ifFalse) + "}" + + // Predefined methods of various types + override def strConcat(left: Ast.expr, right: Ast.expr): String = + "format!(\"{}{}\", " + translate(left) + ", " + translate(right) + ")" + + override def strToInt(s: expr, base: expr): String = + translate(base) match { + case "10" => + s"${translate(s)}.parse().unwrap()" + case _ => + "panic!(\"Converting from string to int in base {} is unimplemented\"" + translate(base) + ")" + } + + override def enumToInt(v: expr, et: EnumType): String = + translate(v) + + override def boolToInt(v: expr): String = + s"${translate(v)} as i32" + + override def floatToInt(v: expr): String = + s"${translate(v)} as i32" + + override def intToStr(i: expr, base: expr): String = { + val baseStr = translate(base) + baseStr match { + case "10" => + s"${translate(i)}.to_string()" + case _ => + s"base_convert(strval(${translate(i)}), 10, $baseStr)" + } + } + override def bytesToStr(bytesExpr: String, encoding: Ast.expr): String = + translate(encoding) match { + case "\"ASCII\"" => + s"String::from_utf8_lossy($bytesExpr)" + case _ => + "panic!(\"Unimplemented encoding for bytesToStr: {}\", " + + translate(encoding) + ")" + } + override def bytesLength(b: Ast.expr): String = + s"${translate(b)}.len()" + override def strLength(s: expr): String = + s"${translate(s)}.len()" + override def strReverse(s: expr): String = + s"${translate(s)}.graphemes(true).rev().flat_map(|g| g.chars()).collect()" + override def strSubstring(s: expr, from: expr, to: expr): String = + s"${translate(s)}.substring(${translate(from)}, ${translate(to)})" + override def strToBytes(s: expr, encoding: expr): String = + "" // TODO: implement + + override def arrayFirst(a: expr): String = + s"${translate(a)}.first()" + override def arrayLast(a: expr): String = + s"${translate(a)}.last()" + override def arraySize(a: expr): String = + s"${translate(a)}.len()" + override def arrayMin(a: Ast.expr): String = + s"${translate(a)}.iter().min()" + override def arrayMax(a: Ast.expr): String = + s"${translate(a)}.iter().max()" + + def types2classAbs(names: List[String]) = + names match { + case List("kaitai_struct") => RustCompiler.kstructName + case _ => RustCompiler.types2classRel(names) + } +} diff --git a/shared/src/main/scala/io/kaitai/struct/translators/TypeDetector.scala b/shared/src/main/scala/io/kaitai/struct/translators/TypeDetector.scala index 1d65e0ba3..17b18d680 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/TypeDetector.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/TypeDetector.scala @@ -3,8 +3,8 @@ package io.kaitai.struct.translators import io.kaitai.struct.datatype.DataType import io.kaitai.struct.datatype.DataType._ import io.kaitai.struct.exprlang.Ast -import io.kaitai.struct.exprlang.Ast.expr -import io.kaitai.struct.precompile.{TypeMismatchError, TypeUndecidedError} +import io.kaitai.struct.format.Identifier +import io.kaitai.struct.precompile.{MethodNotFoundError, TypeMismatchError, TypeUndecidedError} /** * Basic class the implements type inferring functionality for Ast.expr @@ -15,7 +15,26 @@ import io.kaitai.struct.precompile.{TypeMismatchError, TypeUndecidedError} class TypeDetector(provider: TypeProvider) { import TypeDetector._ + /** + * Detects type of a given expression. If it returns a SwitchType, it + * effectively flattens it to a resulting combined type. + * @param v expression + * @return data type + */ def detectType(v: Ast.expr): DataType = { + detectTypeRaw(v) match { + case st: SwitchType => st.combinedType + case other => other + } + } + + /** + * Detects type of a given expression, raw, without any switch type + * flattening. + * @param v expression + * @return data type + */ + def detectTypeRaw(v: Ast.expr): DataType = { v match { case Ast.expr.IntNum(x) => if (x < 0 || x > 255) { @@ -30,13 +49,13 @@ class TypeDetector(provider: TypeProvider) { case Ast.expr.FloatNum(_) => CalcFloatType case Ast.expr.Str(_) => CalcStrType case Ast.expr.Bool(_) => CalcBooleanType - case Ast.expr.EnumByLabel(enumType, _) => + case Ast.expr.EnumByLabel(enumType, _, inType) => val t = EnumType(List(enumType.name), CalcIntType) - t.enumSpec = Some(provider.resolveEnum(enumType.name)) + t.enumSpec = Some(provider.resolveEnum(inType, enumType.name)) t - case Ast.expr.EnumById(enumType, _) => + case Ast.expr.EnumById(enumType, _, inType) => val t = EnumType(List(enumType.name), CalcIntType) - t.enumSpec = Some(provider.resolveEnum(enumType.name)) + t.enumSpec = Some(provider.resolveEnum(inType, enumType.name)) t case Ast.expr.Name(name: Ast.identifier) => provider.determineType(name.name) case Ast.expr.UnaryOp(op: Ast.unaryop, v: Ast.expr) => @@ -71,7 +90,7 @@ class TypeDetector(provider: TypeProvider) { } }) CalcBooleanType - case Ast.expr.IfExp(condition: expr, ifTrue: expr, ifFalse: expr) => + case Ast.expr.IfExp(condition: Ast.expr, ifTrue: Ast.expr, ifFalse: Ast.expr) => detectType(condition) match { case _: BooleanType => val trueType = detectType(ifTrue) @@ -86,80 +105,119 @@ class TypeDetector(provider: TypeProvider) { case _: IntType => elType case idxType => throw new TypeMismatchError(s"unable to index an array using $idxType") } + case _: BytesType => Int1Type(false) case cntType => throw new TypeMismatchError(s"unable to apply operation [] to $cntType") } case Ast.expr.Attribute(value: Ast.expr, attr: Ast.identifier) => - val valType = detectType(value) - valType match { - case KaitaiStructType => - throw new TypeMismatchError(s"called attribute '${attr.name}' on generic struct expression '$value'") - case t: UserType => - t.classSpec match { - case Some(tt) => provider.determineType(tt, attr.name) - case None => throw new TypeUndecidedError(s"expression '$value' has undecided type '${t.name}' (while asking for attribute '${attr.name}')") - } - case _: StrType => - attr.name match { - case "length" => CalcIntType - case "reverse" => CalcStrType - case "to_i" => CalcIntType - case _ => throw new TypeMismatchError(s"called invalid attribute '${attr.name}' on expression of type $valType") - } - case _: IntType => - attr.name match { - case "to_s" => CalcStrType - case _ => throw new TypeMismatchError(s"called invalid attribute '${attr.name}' on expression of type $valType") - } - case ArrayType(inType) => - attr.name match { - case "first" | "last" | "min" | "max" => inType - case "size" => CalcIntType - case _ => throw new TypeMismatchError(s"called invalid attribute '${attr.name}' on expression of type $valType") - } - case _: BytesType => - attr.name match { - case "first" | "last" => Int1Type(false) - case "size" => CalcIntType - case _ => throw new TypeMismatchError(s"called invalid attribute '${attr.name}' on expression of type $valType") - } - case KaitaiStreamType => - attr.name match { - case "size" => CalcIntType - case "pos" => CalcIntType - case "eof" => CalcBooleanType - case _ => throw new TypeMismatchError(s"called invalid attribute '${attr.name}' on expression of type $valType") - } - case et: EnumType => - attr.name match { - case "to_i" => CalcIntType - case _ => throw new TypeMismatchError(s"called invalid attribute '${attr.name}' on expression of type $valType") - } - case _: BooleanType => - attr.name match { - case "to_i" => CalcIntType - case _ => throw new TypeMismatchError(s"called invalid attribute '${attr.name}' on expression of type $valType") - } - case _ => - throw new TypeMismatchError(s"don't know how to call anything on $valType") - } - case Ast.expr.Call(func: Ast.expr, args: Seq[Ast.expr]) => - func match { - case Ast.expr.Attribute(obj: Ast.expr, methodName: Ast.identifier) => - val objType = detectType(obj) - (objType, methodName.name) match { - case (_: StrType, "substring") => CalcStrType - case (_: StrType, "to_i") => CalcIntType - case (_: StrType, "to_b") => CalcBytesType - case _ => throw new RuntimeException(s"don't know how to call method '$methodName' of object type '$objType'") - } - } + detectAttributeType(value, attr) + case call: Ast.expr.Call => + detectCallType(call) case Ast.expr.List(values: Seq[Ast.expr]) => detectArrayType(values) match { case Int1Type(_) => CalcBytesType case t => ArrayType(t) } - case Ast.expr.CastToType(value, typeName) => - provider.resolveType(typeName.name) + case Ast.expr.CastToType(_, typeName) => + detectCastType(typeName) + case Ast.expr.ByteSizeOfType(_) | Ast.expr.BitSizeOfType(_) => + CalcIntType + } + } + + /** + * Detects resulting data type of a given attribute expression. + * + * @note Must be kept in sync with [[CommonMethods.translateAttribute]] + * @param value value part of attribute expression + * @param attr attribute identifier part of attribute expression + * @return data type + */ + def detectAttributeType(value: Ast.expr, attr: Ast.identifier): DataType = { + val valType = detectType(value) + + // Special case: will be compiled as compile-time determined constant + if (attr.name == Identifier.SIZEOF) + return CalcIntType + + valType match { + case KaitaiStructType | CalcKaitaiStructType => + throw new MethodNotFoundError(attr.name, valType) + case t: UserType => + t.classSpec match { + case Some(tt) => provider.determineType(tt, attr.name) + case None => throw new TypeUndecidedError(s"expression '$value' has undecided type '${t.name}' (while asking for attribute '${attr.name}')") + } + case _: BytesType => + attr.name match { + case "length" | "size" => CalcIntType + case "first" | "last" | "min" | "max" => Int1Type(false) + case _ => throw new MethodNotFoundError(attr.name, valType) + } + case _: StrType => + attr.name match { + case "length" => CalcIntType + case "reverse" => CalcStrType + case "to_i" => CalcIntType + case _ => throw new MethodNotFoundError(attr.name, valType) + } + case _: IntType => + attr.name match { + case "to_s" => CalcStrType + case _ => throw new MethodNotFoundError(attr.name, valType) + } + case _: FloatType => + attr.name match { + case "to_i" => CalcIntType + case _ => throw new MethodNotFoundError(attr.name, valType) + } + case ArrayType(inType) => + attr.name match { + case "first" | "last" | "min" | "max" => inType + case "size" => CalcIntType + case _ => throw new MethodNotFoundError(attr.name, valType) + } + case KaitaiStreamType => + attr.name match { + case "size" => CalcIntType + case "pos" => CalcIntType + case "eof" => CalcBooleanType + case _ => throw new MethodNotFoundError(attr.name, valType) + } + case et: EnumType => + attr.name match { + case "to_i" => CalcIntType + case _ => throw new MethodNotFoundError(attr.name, valType) + } + case _: BooleanType => + attr.name match { + case "to_i" => CalcIntType + case _ => throw new MethodNotFoundError(attr.name, valType) + } + case _ => + throw new MethodNotFoundError(attr.name, valType) + } + } + + /** + * Detects resulting data type of a given function call expression. Typical function + * call expression in KSY is `foo.bar(arg1, arg2)`, which is represented in AST as + * `Call(Attribute(foo, bar), Seq(arg1, arg2))`. + * @note Must be kept in sync with [[CommonMethods.translateCall]] + * @param call function call expression + * @return data type + */ + def detectCallType(call: Ast.expr.Call): DataType = { + call.func match { + case Ast.expr.Attribute(obj: Ast.expr, methodName: Ast.identifier) => + val objType = detectType(obj) + // TODO: check number and type of arguments in `call.args` + (objType, methodName.name) match { + case (_: StrType, "substring") => CalcStrType + case (_: StrType, "to_i") => CalcIntType + case (_: StrType, "to_b") => CalcBytesType + case _ => + throw new MethodNotFoundError(methodName.name, objType) + } } } @@ -171,7 +229,7 @@ class TypeDetector(provider: TypeProvider) { * @param values * @return */ - def detectArrayType(values: Seq[expr]): DataType = { + def detectArrayType(values: Seq[Ast.expr]): DataType = { var t1o: Option[DataType] = None values.foreach { v => @@ -187,6 +245,35 @@ class TypeDetector(provider: TypeProvider) { case Some(t) => t } } + + /** + * Detects cast type determined by a typeId definition. + * @param typeName typeId definition to use + * @return data type + */ + def detectCastType(typeName: Ast.typeId): DataType = { + val singleType = if ((!typeName.absolute) && typeName.names.size == 1) { + // May be it's a reserved pure data type name? + DataType.pureFromString(Some(typeName.names(0))) match { + case _: UserType => + // No, it's a user type, let's try to resolve it through provider + provider.resolveType(typeName) + case primitiveType => + // Yes, it is! + primitiveType + } + } else { + // It's a complex type name, it can be only resolved through provider + provider.resolveType(typeName) + } + + // Wrap it in array type, if needed + if (typeName.isArray) { + ArrayType(singleType) + } else { + singleType + } + } } object TypeDetector { @@ -203,9 +290,11 @@ object TypeDetector { case (_: NumericType, _: NumericType) => // ok case (_: BooleanType, _: BooleanType) => // ok case (_: BytesType, _: BytesType) => // ok - case (EnumType(name1, _), EnumType(name2, _)) => - if (name1 != name2) { - throw new TypeMismatchError(s"can't compare different enums '$name1' and '$name2'") + case (et1: EnumType, et2: EnumType) => + val et1Spec = et1.enumSpec.get + val et2Spec = et2.enumSpec.get + if (et1Spec != et2Spec) { + throw new TypeMismatchError(s"can't compare different enums '${et1Spec.nameAsStr}' and '${et2Spec.nameAsStr}'") } op match { case Ast.cmpop.Eq | Ast.cmpop.NotEq => // ok @@ -250,6 +339,7 @@ object TypeDetector { } case (_: IntType, _: IntType) => CalcIntType case (_: NumericType, _: NumericType) => CalcFloatType + case (_: BytesType, _: BytesType) => CalcBytesType case (t1: UserType, t2: UserType) => // Two user types can differ in reserved size and/or processing, but that doesn't matter in case of // type combining - we treat them the same as long as they result in same class spec or have same @@ -260,19 +350,31 @@ object TypeDetector { if (t1.name == t2.name) { t1 } else { - KaitaiStructType + if (t1.isOwning || t2.isOwning) { + KaitaiStructType + } else { + CalcKaitaiStructType + } } case (Some(cs1), Some(cs2)) => if (cs1 == cs2) { t1 } else { - KaitaiStructType + if (t1.isOwning || t2.isOwning) { + KaitaiStructType + } else { + CalcKaitaiStructType + } } case (_, _) => - KaitaiStructType + if (t1.isOwning || t2.isOwning) { + KaitaiStructType + } else { + CalcKaitaiStructType + } } - case (_: UserType, KaitaiStructType) => KaitaiStructType - case (KaitaiStructType, _: UserType) => KaitaiStructType + case (_: UserType, _: ComplexDataType) => CalcKaitaiStructType + case (_: ComplexDataType, _: UserType) => CalcKaitaiStructType case _ => AnyType } } @@ -293,10 +395,53 @@ object TypeDetector { * @return type that can accommodate values of both source types without any data loss */ def combineTypesAndFail(t1: DataType, t2: DataType): DataType = { - combineTypes(t1, t2) match { - case AnyType => - throw new TypeMismatchError(s"can't combine output types: $t1 vs $t2") - case ct => ct + if (t1 == AnyType || t2 == AnyType) { + // combining existing AnyTypes is not a crime :) + AnyType + } else { + combineTypes(t1, t2) match { + case AnyType => + throw new TypeMismatchError(s"can't combine output types: $t1 vs $t2") + case ct => ct + } + } + } + + /** + * Returns true if one can assign value of type `src` into a variable / parameter of type `dst`. + * @param src data type of source value to be assigned + * @param dst destination data type to be assigned into + * @return true if assign if possible + */ + def canAssign(src: DataType, dst: DataType): Boolean = { + if (src == dst) { + // Obviously, if types are equal, they'll fit into one another + true + } else { + (src, dst) match { + case (_, AnyType) => true + case (_: IntType, _: IntType) => true + case (_: FloatType, _: FloatType) => true + case (_: BooleanType, _: BooleanType) => true + case (_: StrType, _: StrType) => true + case (_: UserType, KaitaiStructType) => true + case (_: UserType, CalcKaitaiStructType) => true + case (t1: UserType, t2: UserType) => + (t1.classSpec, t2.classSpec) match { + case (None, None) => + // opaque classes are assignable if their names match + t1.name == t2.name + case (Some(cs1), Some(cs2)) => + // normal user types are assignable if their class specs match + cs1 == cs2 + case (_, _) => + false + } + case (t1: EnumType, t2: EnumType) => + // enums are assignable if their enumSpecs match + t1.enumSpec.get == t2.enumSpec.get + case (_, _) => false + } } } } diff --git a/shared/src/main/scala/io/kaitai/struct/translators/TypeProvider.scala b/shared/src/main/scala/io/kaitai/struct/translators/TypeProvider.scala index 3a9135b4d..1964575ce 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/TypeProvider.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/TypeProvider.scala @@ -1,6 +1,7 @@ package io.kaitai.struct.translators import io.kaitai.struct.datatype.DataType +import io.kaitai.struct.exprlang.Ast import io.kaitai.struct.format.{ClassSpec, EnumSpec} /** @@ -12,6 +13,8 @@ trait TypeProvider { def nowClass: ClassSpec def determineType(attrName: String): DataType def determineType(inClass: ClassSpec, attrName: String): DataType - def resolveEnum(enumName: String): EnumSpec - def resolveType(typeName: String): DataType + def resolveEnum(typeName: Ast.typeId, enumName: String): EnumSpec + def resolveType(typeName: Ast.typeId): DataType + def isLazy(attrName: String): Boolean + def isLazy(inClass: ClassSpec, attrName: String): Boolean } From 2e2e98503deed459084e11044f4e5a22ace17633 Mon Sep 17 00:00:00 2001 From: Petr Pucil Date: Sat, 16 Nov 2019 23:01:54 +0100 Subject: [PATCH 26/90] Stop generating empty constructor when autoRead If someone would compile KSY format with --read-write option only (without --no-auto-read), instancing any user-generated class with empty parameter list would immediately fail with an exception (because autoRead is enabled, but there is no _io to read from). --- .../main/scala/io/kaitai/struct/languages/JavaCompiler.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala index 2d1677001..2a5153d4e 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala @@ -136,7 +136,7 @@ class JavaCompiler(val typeProvider: ClassTypeProvider, config: RuntimeConfig) val paramsRelay = Utils.join(params.map((p) => paramName(p.id)), ", ", ", ", "") - if (config.readWrite) { + if (config.readWrite && !config.autoRead) {// when autoRead is true, it doesn't make sense to generate this constructor because there would be no _io to read from out.puts(s"public ${type2class(name)}(${paramsArg.stripPrefix(", ")}) {") out.inc out.puts(s"this(null, null, null$paramsRelay);") From e46cc950b7d4dbc6cbc5aa8ff35616b94d7ac9b2 Mon Sep 17 00:00:00 2001 From: Petr Pucil Date: Sun, 17 Nov 2019 00:33:25 +0100 Subject: [PATCH 27/90] Use common loop for all repetitions --- .../components/EveryWriteIsExpression.scala | 16 ++++++---------- .../languages/components/GenericChecks.scala | 12 ++++++------ .../languages/components/LanguageCompiler.scala | 3 +++ .../languages/components/UniversalFooter.scala | 2 +- 4 files changed, 16 insertions(+), 17 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala index d5298d4d9..ec27cbd43 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala @@ -17,17 +17,17 @@ trait EveryWriteIsExpression extends LanguageCompiler with ObjectOrientedLanguag attr.cond.repeat match { case RepeatEos => - condRepeatEosHeader2(id, io, attr.dataType, needRaww(attr.dataType)) + condRepeatCommonHeader(id, io, attr.dataType, needRaww(attr.dataType)) attrWrite2(id, attr.dataType, io, attr.cond.repeat, false, defEndian) - condRepeatEosFooter2 + condRepeatCommonFooter case RepeatExpr(repeatExpr: Ast.expr) => - condRepeatExprHeader2(id, io, attr.dataType, needRaww(attr.dataType), repeatExpr) + condRepeatCommonHeader(id, io, attr.dataType, needRaww(attr.dataType)) attrWrite2(id, attr.dataType, io, attr.cond.repeat, false, defEndian) - condRepeatExprFooter + condRepeatCommonFooter case RepeatUntil(untilExpr: Ast.expr) => - condRepeatUntilHeader(id, io, attr.dataType, needRaww(attr.dataType), untilExpr) + condRepeatCommonHeader(id, io, attr.dataType, needRaww(attr.dataType)) attrWrite2(id, attr.dataType, io, attr.cond.repeat, false, defEndian) - condRepeatUntilFooter(id, io, attr.dataType, needRaww(attr.dataType), untilExpr) + condRepeatCommonFooter case NoRepeat => attrWrite2(id, attr.dataType, io, attr.cond.repeat, false, defEndian) } @@ -267,8 +267,4 @@ trait EveryWriteIsExpression extends LanguageCompiler with ObjectOrientedLanguag case _ => false } } - - def condRepeatEosHeader2(id: Identifier, io: String, dataType: DataType, needRaw: Boolean): Unit - def condRepeatEosFooter2: Unit - def condRepeatExprHeader2(id: Identifier, io: String, dataType: DataType, needRaw: Boolean, repeatExpr: Ast.expr): Unit } diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/GenericChecks.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/GenericChecks.scala index 5efd1fdd4..7208e84df 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/GenericChecks.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/GenericChecks.scala @@ -12,18 +12,18 @@ trait GenericChecks extends LanguageCompiler with EveryReadIsExpression with Eve attr.cond.repeat match { case RepeatEos => - condRepeatEosHeader2(id, io, attr.dataType, needRaw(attr.dataType)) + condRepeatCommonHeader(id, io, attr.dataType, needRaw(attr.dataType)) attrCheck2(id, attr.dataType, io, attr.cond.repeat, false) - condRepeatEosFooter2 + condRepeatCommonFooter case RepeatExpr(repeatExpr: Ast.expr) => attrArraySizeCheck(id, repeatExpr) - condRepeatExprHeader2(id, io, attr.dataType, needRaw(attr.dataType), repeatExpr) + condRepeatCommonHeader(id, io, attr.dataType, needRaw(attr.dataType)) attrCheck2(id, attr.dataType, io, attr.cond.repeat, false) - condRepeatExprFooter + condRepeatCommonFooter case RepeatUntil(untilExpr: Ast.expr) => - condRepeatUntilHeader(id, io, attr.dataType, needRaw(attr.dataType), untilExpr) + condRepeatCommonHeader(id, io, attr.dataType, needRaw(attr.dataType)) attrCheck2(id, attr.dataType, io, attr.cond.repeat, false) - condRepeatUntilFooter(id, io, attr.dataType, needRaww(attr.dataType), untilExpr) + condRepeatCommonFooter case NoRepeat => attrCheck2(id, attr.dataType, io, attr.cond.repeat, false) } diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala index cb5ed7436..6a3ce0daa 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala @@ -120,6 +120,9 @@ abstract class LanguageCompiler( def condRepeatUntilHeader(id: Identifier, io: String, dataType: DataType, needRaw: Boolean, repeatExpr: Ast.expr): Unit def condRepeatUntilFooter(id: Identifier, io: String, dataType: DataType, needRaw: Boolean, repeatExpr: Ast.expr): Unit + def condRepeatCommonHeader(id: Identifier, io: String, dataType: DataType, needRaw: Boolean): Unit = {} + def condRepeatCommonFooter: Unit = {} + def attrProcess(proc: ProcessExpr, varSrc: Identifier, varDest: Identifier): Unit def normalIO: String diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/UniversalFooter.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/UniversalFooter.scala index 745e3d0f9..0b3e4ec55 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/UniversalFooter.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/UniversalFooter.scala @@ -20,7 +20,7 @@ trait UniversalFooter extends LanguageCompiler { override def checkFooter: Unit = universalFooter def condRepeatExprFooter = universalFooter def condRepeatEosFooter: Unit = universalFooter - def condRepeatEosFooter2: Unit = universalFooter + override def condRepeatCommonFooter: Unit = universalFooter def condIfFooter(expr: expr): Unit = universalFooter def instanceFooter: Unit = universalFooter } From e4ad9aa1f7d4c8ca81f2223327a934c65a9da9a7 Mon Sep 17 00:00:00 2001 From: Petr Pucil Date: Sat, 14 Dec 2019 14:13:09 +0100 Subject: [PATCH 28/90] Revert "Stop generating empty constructor when autoRead" This reverts commit 2e2e98503deed459084e11044f4e5a22ace17633. --- .../main/scala/io/kaitai/struct/languages/JavaCompiler.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala index 2a5153d4e..2d1677001 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala @@ -136,7 +136,7 @@ class JavaCompiler(val typeProvider: ClassTypeProvider, config: RuntimeConfig) val paramsRelay = Utils.join(params.map((p) => paramName(p.id)), ", ", ", ", "") - if (config.readWrite && !config.autoRead) {// when autoRead is true, it doesn't make sense to generate this constructor because there would be no _io to read from + if (config.readWrite) { out.puts(s"public ${type2class(name)}(${paramsArg.stripPrefix(", ")}) {") out.inc out.puts(s"this(null, null, null$paramsRelay);") From d8adcc5346e22d3305057a1a2e205add55151248 Mon Sep 17 00:00:00 2001 From: Petr Pucil Date: Sun, 2 Feb 2020 20:11:01 +0100 Subject: [PATCH 29/90] Disable autoRead automatically in readWrite mode --- jvm/src/main/scala/io/kaitai/struct/JavaMain.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jvm/src/main/scala/io/kaitai/struct/JavaMain.scala b/jvm/src/main/scala/io/kaitai/struct/JavaMain.scala index b1912dd0a..6b58c7fd1 100644 --- a/jvm/src/main/scala/io/kaitai/struct/JavaMain.scala +++ b/jvm/src/main/scala/io/kaitai/struct/JavaMain.scala @@ -58,7 +58,7 @@ object JavaMain { } opt[Unit]('w', "read-write") action { (x, c) => - c.copy(runtime = c.runtime.copy(readWrite = true)) + c.copy(runtime = c.runtime.copy(readWrite = true, autoRead = false)) } text("generate read-write support in classes (default: read-only)") opt[File]('d', "outdir") valueName("") action { (x, c) => From d26e37b3e6e896cb48cdede76e8a51655fdd1965 Mon Sep 17 00:00:00 2001 From: Petr Pucil Date: Sat, 19 Sep 2020 22:53:14 +0200 Subject: [PATCH 30/90] Extract changes from https://github.com/kaitai-io/kaitai_struct_compiler/pull/183 related to common repetitions --- .../kaitai/struct/languages/JavaCompiler.scala | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala index 2d1677001..cbb2323e4 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala @@ -447,11 +447,6 @@ class JavaCompiler(val typeProvider: ClassTypeProvider, config: RuntimeConfig) importList.add("java.util.ArrayList") } - override def condRepeatEosHeader2(id: Identifier, io: String, dataType: DataType, needRaw: Boolean): Unit = { - out.puts(s"for (int i = 0; i < ${privateMemberName(id)}.size(); i++) {") - out.inc - } - override def handleAssignmentRepeatEos(id: Identifier, expr: String): Unit = { out.puts(s"${privateMemberName(id)}.add($expr);") } @@ -468,16 +463,16 @@ class JavaCompiler(val typeProvider: ClassTypeProvider, config: RuntimeConfig) if (needRaw) out.puts(s"${privateMemberName(RawIdentifier(id))} = new ArrayList(((Number) (${expression(repeatExpr)})).intValue());") out.puts(s"${idToStr(id)} = new ${kaitaiType2JavaType(ArrayType(dataType))}(((Number) (${expression(repeatExpr)})).intValue());") + out.puts(s"for (int i = 0; i < ${expression(repeatExpr)}; i++) {") + out.inc importList.add("java.util.ArrayList") - condRepeatExprHeader2(id, io, dataType, needRaw, repeatExpr) } - override def condRepeatExprHeader2(id: Identifier, io: String, dataType: DataType, needRaw: Boolean, repeatExpr: expr): Unit = { - out.puts(s"for (int i = 0; i < ${expression(repeatExpr)}; i++) {") + // used for all repetitions in _write() and _check() + override def condRepeatCommonHeader(id: Identifier, io: String, dataType: DataType, needRaw: Boolean): Unit = { + out.puts(s"for (int i = 0; i < ${privateMemberName(id)}.size(); i++) {") out.inc - - importList.add("java.util.ArrayList") } override def handleAssignmentRepeatExpr(id: Identifier, expr: String): Unit = { @@ -490,7 +485,7 @@ class JavaCompiler(val typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts(s"${privateMemberName(id)} = new ${kaitaiType2JavaType(ArrayType(dataType))}();") out.puts("{") out.inc - out.puts(s"${kaitaiType2JavaType(dataType)} ${translator.doName("_")};") + out.puts(s"${kaitaiType2JavaType(dataType)} ${translator.doName(Identifier.ITERATOR)};") out.puts("int i = 0;") out.puts("do {") out.inc From 5c70d8fabda39f5aa736a9acf149405d88239b57 Mon Sep 17 00:00:00 2001 From: Petr Pucil Date: Mon, 1 Aug 2022 22:33:29 +0200 Subject: [PATCH 31/90] Fix build errors after upgrading to the 0.10 codebase --- .../io/kaitai/struct/NimClassCompiler.scala | 15 +++--- .../struct/languages/JavaCompiler.scala | 49 ++++++++++--------- .../components/EveryWriteIsExpression.scala | 18 ++----- .../languages/components/GenericChecks.scala | 6 +-- .../components/LanguageCompiler.scala | 2 +- 5 files changed, 43 insertions(+), 47 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/NimClassCompiler.scala b/shared/src/main/scala/io/kaitai/struct/NimClassCompiler.scala index a3f308dfc..b9d9091fb 100644 --- a/shared/src/main/scala/io/kaitai/struct/NimClassCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/NimClassCompiler.scala @@ -51,17 +51,17 @@ class NimClassCompiler( override def compileEagerRead(seq: List[AttrSpec], endian: Option[Endianness]): Unit = { endian match { case None | Some(_: FixedEndian) => - compileSeqProc(seq, None) + compileSeqReadProc(seq, None) case Some(ce: CalcEndian) => - compileSeqProc(seq, Some(LittleEndian)) - compileSeqProc(seq, Some(BigEndian)) + compileSeqReadProc(seq, Some(LittleEndian)) + compileSeqReadProc(seq, Some(BigEndian)) lang.readHeader(None, false) compileCalcEndian(ce) lang.runReadCalc() lang.readFooter() case Some(InheritedEndian) => - compileSeqProc(seq, Some(LittleEndian)) - compileSeqProc(seq, Some(BigEndian)) + compileSeqReadProc(seq, Some(LittleEndian)) + compileSeqReadProc(seq, Some(BigEndian)) lang.readHeader(None, false) lang.runReadCalc() lang.readFooter() @@ -69,7 +69,7 @@ class NimClassCompiler( } // Must override just to add attribute docstrings - override def compileSeq(seq: List[AttrSpec], defEndian: Option[FixedEndian]) = { + override def compileSeqRead(seq: List[AttrSpec], defEndian: Option[FixedEndian]) = { var wasUnaligned = false seq.foreach { (attr) => val nowUnaligned = isUnalignedBits(attr.dataType) @@ -153,7 +153,7 @@ class NimClassCompiler( def compileTypesRec(curClass: ClassSpec): Unit = { curClass.types.foreach { case (_, subClass) => compileTypes(subClass) } } - + // def compileEnumConstants(curClass: ClassSpec): Unit = { // provider.nowClass = curClass // curClass.enums.foreach { case(_, enumColl) => { @@ -203,4 +203,3 @@ class NimClassCompiler( } } - diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala index 2b4e4e0cf..47d8f4192 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala @@ -319,22 +319,26 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) handleAssignment(varDest, expr, rep, false) } - override def attrUnprocess(proc: ProcessExpr, varSrc: Identifier, varDest: Identifier): Unit = { - val srcName = privateMemberName(varSrc) - val destName = privateMemberName(varDest) + // TODO: merge with attrProcess above (there is currently 99.9% duplication) + override def attrUnprocess(proc: ProcessExpr, varSrc: Identifier, varDest: Identifier, rep: RepeatSpec): Unit = { + val srcExpr = getRawIdExpr(varSrc, rep) - proc match { + val expr = proc match { case ProcessXor(xorValue) => - out.puts(s"$destName = $kstreamName.processXor($srcName, ${expression(xorValue)});") + val xorValueStr = translator.detectType(xorValue) match { + case _: IntType => translator.doCast(xorValue, Int1Type(true)) + case _ => expression(xorValue) + } + s"$kstreamName.processXor($srcExpr, $xorValueStr)" case ProcessZlib => - out.puts(s"$destName = $kstreamName.unprocessZlib($srcName);") + s"$kstreamName.unprocessZlib($srcExpr)" case ProcessRotate(isLeft, rotValue) => val expr = if (!isLeft) { expression(rotValue) } else { s"8 - (${expression(rotValue)})" } - out.puts(s"$destName = $kstreamName.processRotateLeft($srcName, $expr, 1);") + s"$kstreamName.processRotateLeft($srcExpr, $expr, 1)" case ProcessCustom(name, args) => val namespace = name.init.mkString(".") val procClass = namespace + @@ -342,8 +346,9 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) type2class(name.last) val procName = s"_process_${idToStr(varSrc)}" out.puts(s"$procClass $procName = new $procClass(${args.map(expression).mkString(", ")});") - out.puts(s"$destName = $procName.encode($srcName);") + s"$procName.encode($srcExpr)" } + handleAssignment(varDest, expr, rep, false) } override def allocateIO(varName: Identifier, rep: RepeatSpec): String = { @@ -483,7 +488,7 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } // used for all repetitions in _write() and _check() - override def condRepeatCommonHeader(id: Identifier, io: String, dataType: DataType, needRaw: Boolean): Unit = { + override def condRepeatCommonHeader(id: Identifier, io: String, dataType: DataType): Unit = { out.puts(s"for (int i = 0; i < ${privateMemberName(id)}.size(); i++) {") out.inc } @@ -837,9 +842,9 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) val stmt = dataType match { case t: ReadableType => s"$io.write${Utils.capitalize(t.apiCall(defEndian))}($expr)" - case BitsType1 => + case BitsType1(bitEndian) => s"$io.writeBitsInt(1, ($expr) ? 1 : 0)" - case BitsType(width: Int) => + case BitsType(width: Int, bitEndian) => s"$io.writeBitsInt($width, $expr)" case _: BytesType => s"$io.writeBytes($expr)" @@ -924,17 +929,6 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) }) } - def idToStr(id: Identifier): String = - id match { - case SpecialIdentifier(name) => name - case NamedIdentifier(name) => Utils.lowerCamelCase(name) - case NumberedIdentifier(idx) => s"_${NumberedIdentifier.TEMPLATE}$idx" - case InstanceIdentifier(name) => Utils.lowerCamelCase(name) - case RawIdentifier(innerId) => s"_raw_${idToStr(innerId)}" - } - - def publicMemberName(id: Identifier) = idToStr(id) - def kaitaiType2JavaType(attrType: DataType): String = kaitaiType2JavaTypePrim(attrType) def kaitaiType2JavaType(attrType: DataType, isNullable: Boolean): String = @@ -1043,6 +1037,17 @@ object JavaCompiler extends LanguageCompilerStatic config: RuntimeConfig ): LanguageCompiler = new JavaCompiler(tp, config) + def idToStr(id: Identifier): String = + id match { + case SpecialIdentifier(name) => name + case NamedIdentifier(name) => Utils.lowerCamelCase(name) + case NumberedIdentifier(idx) => s"_${NumberedIdentifier.TEMPLATE}$idx" + case InstanceIdentifier(name) => Utils.lowerCamelCase(name) + case RawIdentifier(innerId) => s"_raw_${idToStr(innerId)}" + } + + def publicMemberName(id: Identifier) = idToStr(id) + def types2class(names: List[String]) = names.map(x => type2class(x)).mkString(".") override def kstreamName: String = "KaitaiStream" diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala index ec27cbd43..fce8a373c 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala @@ -17,15 +17,15 @@ trait EveryWriteIsExpression extends LanguageCompiler with ObjectOrientedLanguag attr.cond.repeat match { case RepeatEos => - condRepeatCommonHeader(id, io, attr.dataType, needRaww(attr.dataType)) + condRepeatCommonHeader(id, io, attr.dataType) attrWrite2(id, attr.dataType, io, attr.cond.repeat, false, defEndian) condRepeatCommonFooter case RepeatExpr(repeatExpr: Ast.expr) => - condRepeatCommonHeader(id, io, attr.dataType, needRaww(attr.dataType)) + condRepeatCommonHeader(id, io, attr.dataType) attrWrite2(id, attr.dataType, io, attr.cond.repeat, false, defEndian) condRepeatCommonFooter case RepeatUntil(untilExpr: Ast.expr) => - condRepeatCommonHeader(id, io, attr.dataType, needRaww(attr.dataType)) + condRepeatCommonHeader(id, io, attr.dataType) attrWrite2(id, attr.dataType, io, attr.cond.repeat, false, defEndian) condRepeatCommonFooter case NoRepeat => @@ -98,7 +98,7 @@ trait EveryWriteIsExpression extends LanguageCompiler with ObjectOrientedLanguag val idToWrite = t.process match { case Some(proc) => val rawId = RawIdentifier(id) - attrUnprocess(proc, id, rawId) + attrUnprocess(proc, id, rawId, rep) rawId case None => id @@ -258,13 +258,5 @@ trait EveryWriteIsExpression extends LanguageCompiler with ObjectOrientedLanguag def attrWriteStreamToStream(srcIo: String, dstIo: String): Unit def exprStreamToByteArray(ioFixed: String): String - def attrUnprocess(proc: ProcessExpr, varSrc: Identifier, varDest: Identifier): Unit - - // FIXME: unify with EveryReadIsExpression - def needRaww(dataType: DataType): Boolean = { - dataType match { - case t: UserTypeFromBytes => true - case _ => false - } - } + def attrUnprocess(proc: ProcessExpr, varSrc: Identifier, varDest: Identifier, rep: RepeatSpec): Unit } diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/GenericChecks.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/GenericChecks.scala index 7208e84df..2acada974 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/GenericChecks.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/GenericChecks.scala @@ -12,16 +12,16 @@ trait GenericChecks extends LanguageCompiler with EveryReadIsExpression with Eve attr.cond.repeat match { case RepeatEos => - condRepeatCommonHeader(id, io, attr.dataType, needRaw(attr.dataType)) + condRepeatCommonHeader(id, io, attr.dataType) attrCheck2(id, attr.dataType, io, attr.cond.repeat, false) condRepeatCommonFooter case RepeatExpr(repeatExpr: Ast.expr) => attrArraySizeCheck(id, repeatExpr) - condRepeatCommonHeader(id, io, attr.dataType, needRaw(attr.dataType)) + condRepeatCommonHeader(id, io, attr.dataType) attrCheck2(id, attr.dataType, io, attr.cond.repeat, false) condRepeatCommonFooter case RepeatUntil(untilExpr: Ast.expr) => - condRepeatCommonHeader(id, io, attr.dataType, needRaw(attr.dataType)) + condRepeatCommonHeader(id, io, attr.dataType) attrCheck2(id, attr.dataType, io, attr.cond.repeat, false) condRepeatCommonFooter case NoRepeat => diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala index 85375b7c1..528e6c594 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala @@ -123,7 +123,7 @@ abstract class LanguageCompiler( def condRepeatUntilHeader(id: Identifier, io: String, dataType: DataType, untilExpr: Ast.expr): Unit def condRepeatUntilFooter(id: Identifier, io: String, dataType: DataType, untilExpr: Ast.expr): Unit - def condRepeatCommonHeader(id: Identifier, io: String, dataType: DataType, needRaw: Boolean): Unit = {} + def condRepeatCommonHeader(id: Identifier, io: String, dataType: DataType): Unit = {} def condRepeatCommonFooter: Unit = {} def attrProcess(proc: ProcessExpr, varSrc: Identifier, varDest: Identifier, rep: RepeatSpec): Unit From 0867d518b460633276e051a5ac3f07ca68abdfa7 Mon Sep 17 00:00:00 2001 From: Petr Pucil Date: Tue, 16 Aug 2022 12:46:29 +0200 Subject: [PATCH 32/90] Remove `FixedBytesType` as intended in the 0.9 `valid` design --- .../main/scala/io/kaitai/struct/ConstructClassCompiler.scala | 2 -- .../main/scala/io/kaitai/struct/GraphvizClassCompiler.scala | 1 - shared/src/main/scala/io/kaitai/struct/datatype/DataType.scala | 3 +-- .../main/scala/io/kaitai/struct/languages/JavaCompiler.scala | 2 -- .../struct/languages/components/EveryWriteIsExpression.scala | 2 -- .../scala/io/kaitai/struct/languages/components/GoReads.scala | 2 -- .../scala/io/kaitai/struct/precompile/CalculateSeqSizes.scala | 1 - 7 files changed, 1 insertion(+), 12 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/ConstructClassCompiler.scala b/shared/src/main/scala/io/kaitai/struct/ConstructClassCompiler.scala index fa264d553..62fd21176 100644 --- a/shared/src/main/scala/io/kaitai/struct/ConstructClassCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/ConstructClassCompiler.scala @@ -121,8 +121,6 @@ class ConstructClassCompiler(classSpecs: ClassSpecs, topClass: ClassSpec) extend } def typeToStr(dataType: DataType): String = dataType match { - case fbt: FixedBytesType => - s"Const(${translator.doByteArrayLiteral(fbt.contents)})" case Int1Type(signed) => s"Int8${signToStr(signed)}b" case IntMultiType(signed, width, endianOpt) => diff --git a/shared/src/main/scala/io/kaitai/struct/GraphvizClassCompiler.scala b/shared/src/main/scala/io/kaitai/struct/GraphvizClassCompiler.scala index 9d974e0b2..0a59a83fa 100644 --- a/shared/src/main/scala/io/kaitai/struct/GraphvizClassCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/GraphvizClassCompiler.scala @@ -241,7 +241,6 @@ class GraphvizClassCompiler(classSpecs: ClassSpecs, topClass: ClassSpec) extends dataType match { case _: BytesEosType => END_OF_STREAM case blt: BytesLimitType => expressionSize(blt.size, attrName) - case fbt: FixedBytesType => expressionSize(Ast.expr.IntNum(fbt.contents.length), attrName) case StrFromBytesType(basedOn, _) => dataTypeSizeAsString(basedOn, attrName) case utb: UserTypeFromBytes => dataTypeSizeAsString(utb.bytes, attrName) case EnumType(_, basedOn) => dataTypeSizeAsString(basedOn, attrName) diff --git a/shared/src/main/scala/io/kaitai/struct/datatype/DataType.scala b/shared/src/main/scala/io/kaitai/struct/datatype/DataType.scala index 5b7581d8d..9b2a07759 100644 --- a/shared/src/main/scala/io/kaitai/struct/datatype/DataType.scala +++ b/shared/src/main/scala/io/kaitai/struct/datatype/DataType.scala @@ -67,7 +67,6 @@ object DataType { case object CalcBytesType extends BytesType { override def process = None } - case class FixedBytesType(contents: Array[Byte], override val process: Option[ProcessExpr]) extends BytesType case class BytesEosType( terminator: Option[Int], include: Boolean, @@ -323,7 +322,7 @@ object DataType { val r = dto match { case None => arg.contents match { - case Some(c) => FixedBytesType(c, arg.process) + case Some(c) => BytesLimitType(Ast.expr.IntNum(c.length), None, false, None, arg.process) case _ => arg.getByteArrayType(path) } case Some(dt) => dt match { diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala index 47d8f4192..470cad376 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala @@ -544,8 +544,6 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) s"$io.read${Utils.capitalize(t.apiCall(defEndian))}()" case blt: BytesLimitType => s"$io.readBytes(${expression(blt.size)})" - case fbt: FixedBytesType => - s"$io.readBytes(${expression(Ast.expr.IntNum(fbt.contents.length))})" case _: BytesEosType => s"$io.readBytesFull()" case BytesTerminatedType(terminator, include, consume, eosError, _) => diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala index fce8a373c..8c5d0fb1c 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala @@ -104,8 +104,6 @@ trait EveryWriteIsExpression extends LanguageCompiler with ObjectOrientedLanguag id } t match { - case FixedBytesType(contents, process) => - attrPrimitiveWrite(io, translator.doByteArrayLiteral(contents), t, None) case t: BytesEosType => val expr = writeExprAsString(idToWrite, rep, isRaw) attrPrimitiveWrite(io, expr, t, None) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/GoReads.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/GoReads.scala index 1c8bd2c98..17f1d99ae 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/GoReads.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/GoReads.scala @@ -58,8 +58,6 @@ trait GoReads extends CommonReads with ObjectOrientedLanguage with GoSwitchOps { assignType: Option[DataType] = None ): Unit = { dataType match { - case FixedBytesType(c, _) => - attrFixedContentsParse(id, c) case t: UserType => attrUserTypeParse(id, t, io, rep, defEndian) case t: BytesType => diff --git a/shared/src/main/scala/io/kaitai/struct/precompile/CalculateSeqSizes.scala b/shared/src/main/scala/io/kaitai/struct/precompile/CalculateSeqSizes.scala index 77caa2263..61f21ae69 100644 --- a/shared/src/main/scala/io/kaitai/struct/precompile/CalculateSeqSizes.scala +++ b/shared/src/main/scala/io/kaitai/struct/precompile/CalculateSeqSizes.scala @@ -106,7 +106,6 @@ object CalculateSeqSizes { dataType match { case _: Int1Type => FixedSized(1) case IntMultiType(_, width, _) => FixedSized(width.width) - case FixedBytesType(contents, _) => FixedSized(contents.length) case FloatMultiType(width, _) => FixedSized(width.width) case _: BytesEosType => DynamicSized case blt: BytesLimitType => blt.size.evaluateIntConst match { From 939cf98d495b941920753dad379a65346a1d8f2e Mon Sep 17 00:00:00 2001 From: Petr Pucil Date: Mon, 29 Aug 2022 00:06:53 +0200 Subject: [PATCH 33/90] Refactor EveryWriteIsExpression to use Ast.expr.InternalName Closes https://github.com/kaitai-io/kaitai_struct_compiler/pull/181 Now when we have `Ast.expr.InternalName`, I hope I'll never see `Ast.expr.Name(Ast.identifier(idToStr(id)))` again because it's wrong from the ground up and did not generally work, only by a lucky coincidence for the most basic case of named `seq` fields with one-word `id`s (see https://github.com/kaitai-io/kaitai_struct_compiler/pull/181#issue-523952217). Feels good to get rid of it, and as a reward, a number of cases was miraculously fixed thanks to this refactoring. --- .../io/kaitai/struct/ClassTypeProvider.scala | 18 +++++ .../struct/languages/JavaCompiler.scala | 12 +-- .../components/EveryWriteIsExpression.scala | 75 +++++++++---------- .../languages/components/GenericChecks.scala | 28 +------ 4 files changed, 65 insertions(+), 68 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/ClassTypeProvider.scala b/shared/src/main/scala/io/kaitai/struct/ClassTypeProvider.scala index abed472ff..9bc0f7d6f 100644 --- a/shared/src/main/scala/io/kaitai/struct/ClassTypeProvider.scala +++ b/shared/src/main/scala/io/kaitai/struct/ClassTypeProvider.scala @@ -57,6 +57,24 @@ class ClassTypeProvider(classSpecs: ClassSpecs, var topClass: ClassSpec) extends case NamedIdentifier(name) => return determineType(inClass, name) case InstanceIdentifier(name) => return determineType(inClass, name) case SpecialIdentifier(name) => return determineType(inClass, name) + case RawIdentifier(innerId) => { + val innerType = determineType(innerId) + val (isArray, singleType: DataType) = innerType match { + case at: ArrayType => (true, at.elType) + case st: SwitchType => (false, st.cases.collectFirst { + case (_, caseType) + if caseType.isInstanceOf[BytesType] + || caseType.isInstanceOf[UserTypeFromBytes] => caseType + }.get) + case _ => (false, innerType) + } + /** see [[languages.components.ExtraAttrs$]] for possible types */ + val bytesType = singleType match { + case bt: BytesType => bt + case utb: UserTypeFromBytes => utb.bytes + } + return if (isArray) ArrayTypeInStream(bytesType) else bytesType + } case _ => // do nothing } throw new FieldNotFoundError(attrId.humanReadable, inClass) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala index 470cad376..68a7f2142 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala @@ -825,12 +825,13 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def attrPrimitiveWrite( io: String, - exprRaw: String, + valueExpr: Ast.expr, dataType: DataType, defEndian: Option[FixedEndian], exprTypeOpt: Option[DataType] = None ): Unit = { val exprType = exprTypeOpt.getOrElse(dataType) + val exprRaw = expression(valueExpr) val expr = if (exprType != dataType) { s"(${kaitaiType2JavaType(dataType)}) ($exprRaw)" } else { @@ -841,7 +842,7 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) case t: ReadableType => s"$io.write${Utils.capitalize(t.apiCall(defEndian))}($expr)" case BitsType1(bitEndian) => - s"$io.writeBitsInt(1, ($expr) ? 1 : 0)" + s"$io.writeBitsInt(1, ${translator.boolToInt(valueExpr)})" case BitsType(width: Int, bitEndian) => s"$io.writeBitsInt($width, $expr)" case _: BytesType => @@ -850,10 +851,11 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts(stmt + ";") } - override def attrBytesLimitWrite(io: String, expr: String, size: String, term: Int, padRight: Int): Unit = - out.puts(s"$io.writeBytesLimit($expr, $size, (byte) $term, (byte) $padRight);") + override def attrBytesLimitWrite(io: String, expr: Ast.expr, size: String, term: Int, padRight: Int): Unit = + out.puts(s"$io.writeBytesLimit(${expression(expr)}, $size, (byte) $term, (byte) $padRight);") - override def attrUserTypeInstreamWrite(io: String, exprRaw: String, dataType: DataType, exprType: DataType) = { + override def attrUserTypeInstreamWrite(io: String, valueExpr: Ast.expr, dataType: DataType, exprType: DataType) = { + val exprRaw = expression(valueExpr) val expr = if (exprType != dataType) { s"((${kaitaiType2JavaType(dataType)}) ($exprRaw))" } else { diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala index 8c5d0fb1c..dab2c2f9f 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala @@ -4,7 +4,6 @@ import io.kaitai.struct.datatype.DataType import io.kaitai.struct.datatype.DataType._ import io.kaitai.struct.datatype.FixedEndian import io.kaitai.struct.exprlang.Ast -import io.kaitai.struct.exprlang.Ast.expr import io.kaitai.struct.format._ import scala.collection.mutable.ListBuffer @@ -59,30 +58,17 @@ trait EveryWriteIsExpression extends LanguageCompiler with ObjectOrientedLanguag case t: StrFromBytesType => attrStrTypeWrite(id, t, io, rep, isRaw) case t: EnumType => - val expr = translator.enumToInt(Ast.expr.Name(Ast.identifier(idToStr(id))), t) + val expr = writeExprAsExpr(id, rep, isRaw) val exprType = internalEnumIntType(t.basedOn) - attrPrimitiveWrite(io, expr, t.basedOn, defEndian, Some(exprType)) + attrPrimitiveWrite(io, Ast.expr.Attribute(expr, Ast.identifier("to_i")), t.basedOn, defEndian, Some(exprType)) case _ => - val expr = writeExprAsString(id, rep, isRaw) + val expr = writeExprAsExpr(id, rep, isRaw) attrPrimitiveWrite(io, expr, dataType, defEndian, exprTypeOpt) } } - // TODO: unite these methods - def writeExprAsString(id: Identifier, rep: RepeatSpec, isRaw: Boolean): String = { - rep match { - case NoRepeat => - privateMemberName(id) - case _ => - translator.arraySubscript( - Ast.expr.Name(Ast.identifier(idToStr(id))), - Ast.expr.Name(Ast.identifier(Identifier.INDEX)) - ) - } - } - def writeExprAsExpr(id: Identifier, rep: RepeatSpec, isRaw: Boolean): Ast.expr = { - val astId = Ast.expr.Name(Ast.identifier(idToStr(id))) + val astId = Ast.expr.InternalName(id) rep match { case NoRepeat => astId @@ -105,42 +91,46 @@ trait EveryWriteIsExpression extends LanguageCompiler with ObjectOrientedLanguag } t match { case t: BytesEosType => - val expr = writeExprAsString(idToWrite, rep, isRaw) + val expr = writeExprAsExpr(idToWrite, rep, isRaw) attrPrimitiveWrite(io, expr, t, None) - if (t.terminator.isDefined && !t.include) - attrPrimitiveWrite(io, t.terminator.toString, Int1Type(false), None) + t.terminator.foreach { (term) => + if (!t.include) + attrPrimitiveWrite(io, Ast.expr.IntNum(term), Int1Type(false), None) + } case blt: BytesLimitType => - val expr = writeExprAsString(idToWrite, rep, isRaw) + val expr = writeExprAsExpr(idToWrite, rep, isRaw) attrBytesLimitWrite2(io, expr, blt) case t: BytesTerminatedType => - val expr = writeExprAsString(idToWrite, rep, isRaw) + val expr = writeExprAsExpr(idToWrite, rep, isRaw) attrPrimitiveWrite(io, expr, t, None) if (t.consume && !t.include) - attrPrimitiveWrite(io, t.terminator.toString, Int1Type(false), None) + attrPrimitiveWrite(io, Ast.expr.IntNum(t.terminator), Int1Type(false), None) } } def attrStrTypeWrite(id: Identifier, t: StrFromBytesType, io: String, rep: RepeatSpec, isRaw: Boolean) = { - val expr = translator.strToBytes(writeExprAsExpr(id, rep, isRaw), Ast.expr.Str(t.encoding)) + val expr = exprStrToBytes(writeExprAsExpr(id, rep, isRaw), t.encoding) attrPrimitiveWrite(io, expr, t.bytes, None) t.bytes match { case t: BytesEosType => - if (t.terminator.isDefined && !t.include) - attrPrimitiveWrite(io, t.terminator.toString, Int1Type(false), None) + t.terminator.foreach { (term) => + if (!t.include) + attrPrimitiveWrite(io, Ast.expr.IntNum(term), Int1Type(false), None) + } case t: BytesLimitType => // FIXME: implement padding and terminator byte - t.terminator.foreach((terminator) => + t.terminator.foreach { (term) => if (!t.include) - attrPrimitiveWrite(io, terminator.toString, Int1Type(false), None) - ) + attrPrimitiveWrite(io, Ast.expr.IntNum(term), Int1Type(false), None) + } case t: BytesTerminatedType => if (t.consume && !t.include) - attrPrimitiveWrite(io, t.terminator.toString, Int1Type(false), None) + attrPrimitiveWrite(io, Ast.expr.IntNum(t.terminator), Int1Type(false), None) } } - def attrBytesLimitWrite2(io: String, expr: String, blt: BytesLimitType): Unit = { + def attrBytesLimitWrite2(io: String, expr: Ast.expr, blt: BytesLimitType): Unit = { val (term, padRight) = (blt.terminator, blt.padRight, blt.include) match { case (None, None, false) => // no terminator, no padding => just a regular output @@ -177,7 +167,7 @@ trait EveryWriteIsExpression extends LanguageCompiler with ObjectOrientedLanguag exprTypeOpt: Option[DataType] = None ) = { val exprType = exprTypeOpt.getOrElse(t) - val expr = writeExprAsString(id, rep, isRaw) + val expr = writeExprAsExpr(id, rep, isRaw) t match { case _: UserTypeInstream => @@ -218,8 +208,8 @@ trait EveryWriteIsExpression extends LanguageCompiler with ObjectOrientedLanguag } def attrSwitchTypeWrite( id: Identifier, - on: expr, - cases: Map[expr, DataType], + on: Ast.expr, + cases: Map[Ast.expr, DataType], io: String, rep: RepeatSpec, defEndian: Option[FixedEndian], @@ -248,11 +238,20 @@ trait EveryWriteIsExpression extends LanguageCompiler with ObjectOrientedLanguag ) } + def exprStrToBytes(name: Ast.expr, encoding: String) = + Ast.expr.Call( + Ast.expr.Attribute( + name, + Ast.identifier("to_b") + ), + Seq(Ast.expr.Str(encoding)) + ) + def internalEnumIntType(basedOn: IntType): DataType - def attrPrimitiveWrite(io: String, expr: String, dt: DataType, defEndian: Option[FixedEndian], exprTypeOpt: Option[DataType] = None): Unit - def attrBytesLimitWrite(io: String, expr: String, size: String, term: Int, padRight: Int): Unit - def attrUserTypeInstreamWrite(io: String, expr: String, t: DataType, exprType: DataType): Unit + def attrPrimitiveWrite(io: String, expr: Ast.expr, dt: DataType, defEndian: Option[FixedEndian], exprTypeOpt: Option[DataType] = None): Unit + def attrBytesLimitWrite(io: String, expr: Ast.expr, size: String, term: Int, padRight: Int): Unit + def attrUserTypeInstreamWrite(io: String, expr: Ast.expr, t: DataType, exprType: DataType): Unit def attrWriteStreamToStream(srcIo: String, dstIo: String): Unit def exprStreamToByteArray(ioFixed: String): String diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/GenericChecks.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/GenericChecks.scala index 2acada974..9181dd081 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/GenericChecks.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/GenericChecks.scala @@ -32,21 +32,10 @@ trait GenericChecks extends LanguageCompiler with EveryReadIsExpression with Eve } def attrCheck2(id: Identifier, dataType: DataType, io: String, repeat: RepeatSpec, isRaw: Boolean) = { - // TODO: unify with EveryWriteIsExpression.writeExprAsExpr - val astName = idToName(id) - val item = repeat match { - case NoRepeat => - astName - case _ => - Ast.expr.Subscript( - astName, - Ast.expr.Name(Ast.identifier(Identifier.INDEX)) - ) - } - + val item = writeExprAsExpr(id, repeat, isRaw) dataType match { case t: BytesType => - attrByteSizeCheck(astName, t, exprByteArraySize(item), idToMsg(id)) + attrByteSizeCheck(Ast.expr.InternalName(id), t, exprByteArraySize(item), idToMsg(id)) case st: StrFromBytesType => val bytes = exprStrToBytes(item, st.encoding) attrByteSizeCheck( @@ -61,7 +50,7 @@ trait GenericChecks extends LanguageCompiler with EveryReadIsExpression with Eve def attrArraySizeCheck(id: Identifier, expectedSize: Ast.expr): Unit = attrAssertEqual( - exprArraySize(idToName(id)), + exprArraySize(Ast.expr.InternalName(id)), expectedSize, idToMsg(id) ) @@ -116,8 +105,6 @@ trait GenericChecks extends LanguageCompiler with EveryReadIsExpression with Eve case SpecialIdentifier(name) => name } - def idToName(id: Identifier): Ast.expr.Name = Ast.expr.Name(Ast.identifier(idToMsg(id))) - def exprByteArraySize(name: Ast.expr) = Ast.expr.Attribute( name, @@ -126,15 +113,6 @@ trait GenericChecks extends LanguageCompiler with EveryReadIsExpression with Eve def exprArraySize(name: Ast.expr) = exprByteArraySize(name) - def exprStrToBytes(name: Ast.expr, encoding: String) = - Ast.expr.Call( - Ast.expr.Attribute( - name, - Ast.identifier("to_b") - ), - Seq(Ast.expr.Str(encoding)) - ) - def attrAssertLastByte(name: Ast.expr, expectedLast: Int, msg: String): Unit = { attrAssertEqual( Ast.expr.Attribute( From 83b1fa74dfb32b7b8b9b6d8361a8ceef38394288 Mon Sep 17 00:00:00 2001 From: Petr Pucil Date: Sun, 4 Sep 2022 12:27:20 +0200 Subject: [PATCH 34/90] Delete unused attrFixedContentsParse (_io.ensure_fixed_contents()) --- .../struct/languages/CSharpCompiler.scala | 4 ---- .../kaitai/struct/languages/CppCompiler.scala | 4 ---- .../kaitai/struct/languages/GoCompiler.scala | 19 ----------------- .../struct/languages/JavaCompiler.scala | 5 ----- .../struct/languages/JavaScriptCompiler.scala | 8 +------ .../kaitai/struct/languages/LuaCompiler.scala | 4 ---- .../kaitai/struct/languages/NimCompiler.scala | 4 ---- .../kaitai/struct/languages/PHPCompiler.scala | 4 ---- .../struct/languages/PerlCompiler.scala | 5 ----- .../struct/languages/PythonCompiler.scala | 4 ---- .../struct/languages/RubyCompiler.scala | 4 ---- .../struct/languages/RustCompiler.scala | 4 ---- .../FixedContentsUsingArrayByteLiteral.scala | 21 ------------------- .../components/LanguageCompiler.scala | 3 --- 14 files changed, 1 insertion(+), 92 deletions(-) delete mode 100644 shared/src/main/scala/io/kaitai/struct/languages/components/FixedContentsUsingArrayByteLiteral.scala diff --git a/shared/src/main/scala/io/kaitai/struct/languages/CSharpCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/CSharpCompiler.scala index 1654a4ae0..733aaa2ef 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/CSharpCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/CSharpCompiler.scala @@ -17,7 +17,6 @@ class CSharpCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) with AllocateIOLocalVar with EveryReadIsExpression with UniversalDoc - with FixedContentsUsingArrayByteLiteral with SwitchIfOps with NoNeedForFullClassPath { import CSharpCompiler._ @@ -193,9 +192,6 @@ class CSharpCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts("}") } - override def attrFixedContentsParse(attrName: Identifier, contents: String): Unit = - out.puts(s"${privateMemberName(attrName)} = $normalIO.EnsureFixedContents($contents);") - override def attrProcess(proc: ProcessExpr, varSrc: Identifier, varDest: Identifier, rep: RepeatSpec): Unit = { val srcExpr = getRawIdExpr(varSrc, rep) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala index 58d79b1bf..ff668fceb 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala @@ -16,7 +16,6 @@ class CppCompiler( ) extends LanguageCompiler(typeProvider, config) with ObjectOrientedLanguage with AllocateAndStoreIO - with FixedContentsUsingArrayByteLiteral with UniversalDoc with SwitchIfOps with EveryReadIsExpression { @@ -496,9 +495,6 @@ class CppCompiler( outSrc.puts("}") } - override def attrFixedContentsParse(attrName: Identifier, contents: String): Unit = - outSrc.puts(s"${privateMemberName(attrName)} = $normalIO->ensure_fixed_contents($contents);") - override def attrProcess(proc: ProcessExpr, varSrc: Identifier, varDest: Identifier, rep: RepeatSpec): Unit = { val srcExpr = getRawIdExpr(varSrc, rep) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/GoCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/GoCompiler.scala index 93ed4de2c..b0e399bf6 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/GoCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/GoCompiler.scala @@ -190,25 +190,6 @@ class GoCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts("}") } - override def attrFixedContentsParse(attrName: Identifier, contents: Array[Byte]): Unit = { - out.puts(s"${privateMemberName(attrName)}, err = $normalIO.ReadBytes(${contents.length})") - - out.puts(s"if err != nil {") - out.inc - out.puts("return err") - out.dec - out.puts("}") - - importList.add("bytes") - importList.add("errors") - val expected = translator.resToStr(translator.doByteArrayLiteral(contents)) - out.puts(s"if !bytes.Equal(${privateMemberName(attrName)}, $expected) {") - out.inc - out.puts("return errors.New(\"Unexpected fixed contents\")") - out.dec - out.puts("}") - } - override def attrProcess(proc: ProcessExpr, varSrc: Identifier, varDest: Identifier, rep: RepeatSpec): Unit = { val srcExpr = getRawIdExpr(varSrc, rep) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala index 68a7f2142..2c20f3810 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala @@ -20,7 +20,6 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) with UniversalFooter with UniversalDoc with AllocateIOLocalVar - with FixedContentsUsingArrayByteLiteral with SwitchIfOps with NoNeedForFullClassPath { import JavaCompiler._ @@ -284,10 +283,6 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts("}") } - override def attrFixedContentsParse(attrName: Identifier, contents: String): Unit = { - out.puts(s"${privateMemberName(attrName)} = $normalIO.ensureFixedContents($contents);") - } - override def attrProcess(proc: ProcessExpr, varSrc: Identifier, varDest: Identifier, rep: RepeatSpec): Unit = { val srcExpr = getRawIdExpr(varSrc, rep) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JavaScriptCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JavaScriptCompiler.scala index cf7cbaf0c..7833db8d5 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/JavaScriptCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/JavaScriptCompiler.scala @@ -17,8 +17,7 @@ class JavaScriptCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) with UniversalDoc with AllocateIOLocalVar with EveryReadIsExpression - with SwitchIfOps - with FixedContentsUsingArrayByteLiteral { + with SwitchIfOps { import JavaScriptCompiler._ override val translator = new JavaScriptTranslator(typeProvider) @@ -184,11 +183,6 @@ class JavaScriptCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts("}") } - override def attrFixedContentsParse(attrName: Identifier, contents: String): Unit = { - out.puts(s"${privateMemberName(attrName)} = " + - s"$normalIO.ensureFixedContents($contents);") - } - override def attrProcess(proc: ProcessExpr, varSrc: Identifier, varDest: Identifier, rep: RepeatSpec): Unit = { val srcExpr = getRawIdExpr(varSrc, rep) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/LuaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/LuaCompiler.scala index ab33119f1..e83c30ec9 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/LuaCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/LuaCompiler.scala @@ -12,7 +12,6 @@ class LuaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) extends LanguageCompiler(typeProvider, config) with AllocateIOLocalVar with EveryReadIsExpression - with FixedContentsUsingArrayByteLiteral with ObjectOrientedLanguage with SingleOutputFile with UniversalDoc @@ -150,9 +149,6 @@ class LuaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts("end") } - override def attrFixedContentsParse(attrName: Identifier, contents: String): Unit = - out.puts(s"${privateMemberName(attrName)} = self._io:ensure_fixed_contents($contents)") - override def condIfHeader(expr: Ast.expr): Unit = { out.puts(s"if ${expression(expr)} then") out.inc diff --git a/shared/src/main/scala/io/kaitai/struct/languages/NimCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/NimCompiler.scala index 3fc4d0e07..06a854d6e 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/NimCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/NimCompiler.scala @@ -13,7 +13,6 @@ class NimCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) with SingleOutputFile with EveryReadIsExpression with UpperCamelCaseClasses - with FixedContentsUsingArrayByteLiteral with UniversalFooter with AllocateIOLocalVar with SwitchIfOps @@ -77,9 +76,6 @@ class NimCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) importList.add(file) } override def alignToByte(io: String): Unit = out.puts(s"alignToByte($io)") - override def attrFixedContentsParse(attrName: Identifier, contents: String): Unit = { - out.puts(s"this.${idToStr(attrName)} = $normalIO.ensureFixedContents($contents)") - } // def attrParse(attr: AttrLikeSpec, id: Identifier, defEndian: Option[Endianness]): Unit = ??? override def attrParseHybrid(leProc: () => Unit, beProc: () => Unit): Unit = { out.puts("if this.isLe:") diff --git a/shared/src/main/scala/io/kaitai/struct/languages/PHPCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/PHPCompiler.scala index 785234278..7947c7a47 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/PHPCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/PHPCompiler.scala @@ -16,7 +16,6 @@ class PHPCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) with AllocateIOLocalVar with UniversalFooter with UniversalDoc - with FixedContentsUsingArrayByteLiteral with EveryReadIsExpression { import PHPCompiler._ @@ -191,9 +190,6 @@ class PHPCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts("}") } - override def attrFixedContentsParse(attrName: Identifier, contents: String): Unit = - out.puts(s"${privateMemberName(attrName)} = $normalIO->ensureFixedContents($contents);") - override def attrProcess(proc: ProcessExpr, varSrc: Identifier, varDest: Identifier, rep: RepeatSpec): Unit = { val srcExpr = getRawIdExpr(varSrc, rep) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/PerlCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/PerlCompiler.scala index ffedf96de..2beeffd72 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/PerlCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/PerlCompiler.scala @@ -15,7 +15,6 @@ class PerlCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) with UniversalFooter with UpperCamelCaseClasses with AllocateIOLocalVar - with FixedContentsUsingArrayByteLiteral with SwitchIfOps with EveryReadIsExpression { @@ -163,10 +162,6 @@ class PerlCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts("}") } - override def attrFixedContentsParse(attrName: Identifier, contents: String): Unit = { - out.puts(s"${privateMemberName(attrName)} = $normalIO->ensure_fixed_contents($contents);") - } - override def attrProcess(proc: ProcessExpr, varSrc: Identifier, varDest: Identifier, rep: RepeatSpec): Unit = { val srcExpr = getRawIdExpr(varSrc, rep) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/PythonCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/PythonCompiler.scala index 3b31eb420..a97c86daa 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/PythonCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/PythonCompiler.scala @@ -17,7 +17,6 @@ class PythonCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) with UniversalFooter with EveryReadIsExpression with AllocateIOLocalVar - with FixedContentsUsingArrayByteLiteral with UniversalDoc with SwitchIfOps with NoNeedForFullClassPath { @@ -174,9 +173,6 @@ class PythonCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.putsLines("", "\"\"\"" + docStr + refStr + "\"\"\"") } - override def attrFixedContentsParse(attrName: Identifier, contents: String): Unit = - out.puts(s"${privateMemberName(attrName)} = self._io.ensure_fixed_contents($contents)") - override def attrParseHybrid(leProc: () => Unit, beProc: () => Unit): Unit = { out.puts("if self._is_le:") out.inc diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RubyCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RubyCompiler.scala index 25c9bc3c8..b5373c072 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RubyCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RubyCompiler.scala @@ -18,7 +18,6 @@ class RubyCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) with UpperCamelCaseClasses with AllocateIOLocalVar with EveryReadIsExpression - with FixedContentsUsingArrayByteLiteral with NoNeedForFullClassPath { import RubyCompiler._ @@ -174,9 +173,6 @@ class RubyCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts("end") } - override def attrFixedContentsParse(attrName: Identifier, contents: String): Unit = - out.puts(s"${privateMemberName(attrName)} = $normalIO.ensure_fixed_contents($contents)") - override def attrProcess(proc: ProcessExpr, varSrc: Identifier, varDest: Identifier, rep: RepeatSpec): Unit = { val srcExpr = getRawIdExpr(varSrc, rep) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala index 2a460879a..2f7a118ed 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala @@ -16,7 +16,6 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) with AllocateIOLocalVar with UniversalFooter with UniversalDoc - with FixedContentsUsingArrayByteLiteral with EveryReadIsExpression { import RustCompiler._ @@ -170,9 +169,6 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts("}") } - override def attrFixedContentsParse(attrName: Identifier, contents: String): Unit = - out.puts(s"${privateMemberName(attrName)} = $normalIO.ensureFixedContents($contents);") - override def attrProcess(proc: ProcessExpr, varSrc: Identifier, varDest: Identifier, rep: RepeatSpec): Unit = { val srcExpr = getRawIdExpr(varSrc, rep) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/FixedContentsUsingArrayByteLiteral.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/FixedContentsUsingArrayByteLiteral.scala deleted file mode 100644 index c5d4abd9b..000000000 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/FixedContentsUsingArrayByteLiteral.scala +++ /dev/null @@ -1,21 +0,0 @@ -package io.kaitai.struct.languages.components - -import io.kaitai.struct.exprlang.Ast -import io.kaitai.struct.format.Identifier - -/** - * Allows uniform implementation of attrFixedContentsParse by enforcing usage - * of doByteArrayLiteral in relevant language's translator. - */ -trait FixedContentsUsingArrayByteLiteral extends LanguageCompiler { - def attrFixedContentsParse(attrName: Identifier, contents: Array[Byte]) = - attrFixedContentsParse( - attrName, - translator.translate( - Ast.expr.List( - contents.map(x => Ast.expr.IntNum(BigInt(x & 0xff))) - ) - ) - ) - def attrFixedContentsParse(attrName: Identifier, contents: String): Unit -} diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala index 528e6c594..c1e8af362 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala @@ -104,9 +104,6 @@ abstract class LanguageCompiler( def checkFooter(): Unit = ??? def attrCheck(attr: AttrLikeSpec, id: Identifier): Unit = ??? - // TODO: delete - def attrFixedContentsParse(attrName: Identifier, contents: Array[Byte]): Unit - def condIfSetNull(instName: Identifier): Unit = {} def condIfSetNonNull(instName: Identifier): Unit = {} def condIfHeader(expr: Ast.expr): Unit From caea2b72f5bb58eedb9e646a74f1c890c62b4b98 Mon Sep 17 00:00:00 2001 From: Petr Pucil Date: Sun, 4 Sep 2022 14:45:36 +0200 Subject: [PATCH 35/90] Fix writing of a field with `terminator` + `consume: false` --- .../components/EveryWriteIsExpression.scala | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala index dab2c2f9f..396da4b4a 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala @@ -103,8 +103,17 @@ trait EveryWriteIsExpression extends LanguageCompiler with ObjectOrientedLanguag case t: BytesTerminatedType => val expr = writeExprAsExpr(idToWrite, rep, isRaw) attrPrimitiveWrite(io, expr, t, None) - if (t.consume && !t.include) + if (!t.include) { + if (!t.consume) { + blockScopeHeader + pushPos(io) + } attrPrimitiveWrite(io, Ast.expr.IntNum(t.terminator), Int1Type(false), None) + if (!t.consume) { + popPos(io) + blockScopeFooter + } + } } } @@ -112,6 +121,7 @@ trait EveryWriteIsExpression extends LanguageCompiler with ObjectOrientedLanguag val expr = exprStrToBytes(writeExprAsExpr(id, rep, isRaw), t.encoding) attrPrimitiveWrite(io, expr, t.bytes, None) + /** FIXME: duplication with the previous [[attrBytesTypeWrite]] method (will also ensure consistent handling) */ t.bytes match { case t: BytesEosType => t.terminator.foreach { (term) => From c1d391a6ed6ebbabefea6016d3ac1828c6cf6fc4 Mon Sep 17 00:00:00 2001 From: Petr Pucil Date: Mon, 5 Sep 2022 16:41:48 +0200 Subject: [PATCH 36/90] Fix broken consistency check for repeated byte arrays See https://github.com/kaitai-io/kaitai_struct_tests/commit/0335f1011cd613f5733c90f710797e1eed26ee9d for reference. --- .../io/kaitai/struct/languages/components/GenericChecks.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/GenericChecks.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/GenericChecks.scala index 9181dd081..cb73b9324 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/GenericChecks.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/GenericChecks.scala @@ -35,7 +35,7 @@ trait GenericChecks extends LanguageCompiler with EveryReadIsExpression with Eve val item = writeExprAsExpr(id, repeat, isRaw) dataType match { case t: BytesType => - attrByteSizeCheck(Ast.expr.InternalName(id), t, exprByteArraySize(item), idToMsg(id)) + attrByteSizeCheck(item, t, exprByteArraySize(item), idToMsg(id)) case st: StrFromBytesType => val bytes = exprStrToBytes(item, st.encoding) attrByteSizeCheck( From 43d044a563ffcbc3af2c578a8fa8d53161b9c8b8 Mon Sep 17 00:00:00 2001 From: Petr Pucil Date: Mon, 5 Sep 2022 16:48:02 +0200 Subject: [PATCH 37/90] Java: fix bytesSubscript to give unsigned byte values (finally!) See https://github.com/kaitai-io/kaitai_struct/issues/633 --- .../kaitai/struct/translators/JavaTranslator.scala | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala index a76c40859..8a7c44c5f 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala @@ -129,11 +129,18 @@ class JavaTranslator(provider: TypeProvider, importList: ImportList, config: Run override def bytesLength(b: Ast.expr): String = s"${translate(b)}.length" override def bytesSubscript(container: Ast.expr, idx: Ast.expr): String = - s"${translate(container)}[${translate(idx)}]" + s"(${translate(container)}[${translate(idx)}] & 0xff)" override def bytesFirst(b: Ast.expr): String = - s"${translate(b)}[0]" + bytesSubscript(b, Ast.expr.IntNum(0)) override def bytesLast(b: Ast.expr): String = - s"${translate(b)}[(${translate(b)}).length - 1]" + bytesSubscript(b, Ast.expr.BinOp( + Ast.expr.Attribute( + b, + Ast.identifier("length") + ), + Ast.operator.Sub, + Ast.expr.IntNum(1) + )) override def bytesMin(b: Ast.expr): String = s"${JavaCompiler.kstreamName}.byteArrayMin(${translate(b)})" override def bytesMax(b: Ast.expr): String = From 840431958199c7d9a0049be3c4e2e0cbd39f59d8 Mon Sep 17 00:00:00 2001 From: Petr Pucil Date: Thu, 8 Sep 2022 11:10:57 +0200 Subject: [PATCH 38/90] Drop `expr` argument of condIfFooter method --- .../scala/io/kaitai/struct/languages/CSharpCompiler.scala | 2 +- .../main/scala/io/kaitai/struct/languages/CppCompiler.scala | 2 +- .../scala/io/kaitai/struct/languages/JavaScriptCompiler.scala | 2 +- .../main/scala/io/kaitai/struct/languages/LuaCompiler.scala | 2 +- .../kaitai/struct/languages/components/LanguageCompiler.scala | 4 ++-- .../kaitai/struct/languages/components/UniversalFooter.scala | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/CSharpCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/CSharpCompiler.scala index 733aaa2ef..0d63b8ce1 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/CSharpCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/CSharpCompiler.scala @@ -268,7 +268,7 @@ class CSharpCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.inc } - override def condIfFooter(expr: expr): Unit = fileFooter(null) + override def condIfFooter: Unit = fileFooter(null) override def condRepeatCommonInit(id: Identifier, dataType: DataType, needRaw: NeedRaw): Unit = { importList.add("System.Collections.Generic") diff --git a/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala index ff668fceb..e32985878 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala @@ -610,7 +610,7 @@ class CppCompiler( outSrc.inc } - override def condIfFooter(expr: Ast.expr): Unit = { + override def condIfFooter: Unit = { outSrc.dec outSrc.puts("}") } diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JavaScriptCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JavaScriptCompiler.scala index 7833db8d5..ee29bb8ca 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/JavaScriptCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/JavaScriptCompiler.scala @@ -286,7 +286,7 @@ class JavaScriptCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } // TODO: replace this with UniversalFooter - override def condIfFooter(expr: expr): Unit = { + override def condIfFooter: Unit = { out.dec out.puts("}") } diff --git a/shared/src/main/scala/io/kaitai/struct/languages/LuaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/LuaCompiler.scala index e83c30ec9..abd2a537b 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/LuaCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/LuaCompiler.scala @@ -153,7 +153,7 @@ class LuaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts(s"if ${expression(expr)} then") out.inc } - override def condIfFooter(expr: Ast.expr): Unit = { + override def condIfFooter: Unit = { out.dec out.puts("end") } diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala index c1e8af362..8f6178919 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala @@ -107,7 +107,7 @@ abstract class LanguageCompiler( def condIfSetNull(instName: Identifier): Unit = {} def condIfSetNonNull(instName: Identifier): Unit = {} def condIfHeader(expr: Ast.expr): Unit - def condIfFooter(expr: Ast.expr): Unit + def condIfFooter: Unit def condRepeatCommonInit(id: Identifier, dataType: DataType, needRaw: NeedRaw): Unit @@ -173,7 +173,7 @@ abstract class LanguageCompiler( def attrParseIfFooter(ifExpr: Option[Ast.expr]): Unit = { ifExpr match { - case Some(e) => condIfFooter(e) + case Some(e) => condIfFooter case None => // ignore } } diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/UniversalFooter.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/UniversalFooter.scala index 0b3e4ec55..c1e3895c6 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/UniversalFooter.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/UniversalFooter.scala @@ -21,6 +21,6 @@ trait UniversalFooter extends LanguageCompiler { def condRepeatExprFooter = universalFooter def condRepeatEosFooter: Unit = universalFooter override def condRepeatCommonFooter: Unit = universalFooter - def condIfFooter(expr: expr): Unit = universalFooter + def condIfFooter: Unit = universalFooter def instanceFooter: Unit = universalFooter } From 2acf924341127f301f17fa5396ea90e601987a9d Mon Sep 17 00:00:00 2001 From: Petr Pucil Date: Thu, 8 Sep 2022 12:44:51 +0200 Subject: [PATCH 39/90] Java: include term if `include: true` && `terminator` == `pad-right` TODO: do this in all target languages Related: https://github.com/kaitai-io/kaitai_struct/issues/705 If you have a field with `include: true` and both `terminator` and `pad-right`, whether the terminator byte will actually be present in the field value should not depend on whether `terminator` == `pad-right` or `terminator` != `pad-right`. Currently it does - consider field bytes `D D D T P P`, where `D` mean "data" bytes, `T` is the byte specified by `terminator` and `P` is a padding specified by `pad-right`. If `T == P`, bytesStripRight will yield just `D D D`, so the terminator byte `T` has been lost (because it looked like a part of padding) and bytesTerminate won't be able to include it (even though the `include: true` argument is set), since it hasn't found it and returns `D D D` as well. In contrast, if `T != P`, bytesStripRight returns `D D D T`, and bytesTerminate will scan these bytes for the terminator, finds it at the end and includes it, so yields `D D D T` as well. The solution is to skip applying bytesStripRight if `terminator` == `pad-right` and pass the original bytes directly to bytesTerminate. This fix makes no change for `include: false` outwardly (only internally - now it will be a deliberate decision of bytesTerminate not to include the terminator, instead of being mistakenly swallowed up by bytesStripRight and bytesTerminate then having no say in including the terminator), only for `include: true`. --- .../main/scala/io/kaitai/struct/languages/JavaCompiler.scala | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala index 2c20f3810..78684c09b 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala @@ -575,8 +575,9 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def bytesPadTermExpr(expr0: String, padRight: Option[Int], terminator: Option[Int], include: Boolean) = { val expr1 = padRight match { - case Some(padByte) => s"$kstreamName.bytesStripRight($expr0, (byte) $padByte)" - case None => expr0 + case Some(padByte) if terminator.map(term => padByte != term).getOrElse(true) => + s"$kstreamName.bytesStripRight($expr0, (byte) $padByte)" + case _ => expr0 } val expr2 = terminator match { case Some(term) => s"$kstreamName.bytesTerminate($expr1, (byte) $term, $include)" From 73b034703f693c4c46353ca77dbebf73bef7e528 Mon Sep 17 00:00:00 2001 From: Petr Pucil Date: Sat, 10 Sep 2022 22:26:49 +0200 Subject: [PATCH 40/90] Fix & add missing `terminator`/`pad-right` consistency checks Depends on https://github.com/kaitai-io/kaitai_struct_java_runtime/commit/93db9fef854567860f02f51f08185ac595c741e7 --- .../struct/languages/JavaCompiler.scala | 8 +- .../components/EveryWriteIsExpression.scala | 2 + .../languages/components/GenericChecks.scala | 158 +++++++++++++----- .../struct/translators/CommonMethods.scala | 2 + .../struct/translators/JavaTranslator.scala | 2 + .../struct/translators/TypeDetector.scala | 1 + 6 files changed, 123 insertions(+), 50 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala index 78684c09b..f4beb23ff 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala @@ -866,12 +866,12 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def exprStreamToByteArray(io: String): String = s"$io.toByteArray()" - override def attrBasicCheck(checkExpr: String, actual: String, expected: String, msg: String): Unit = { - val msgStr = translator.translate(Ast.expr.Str(msg)) + override def attrBasicCheck(checkExpr: Ast.expr, actual: Ast.expr, expected: Ast.expr, msg: String): Unit = { + val msgStr = expression(Ast.expr.Str(msg)) - out.puts(s"if ($checkExpr)") + out.puts(s"if (${expression(checkExpr)})") out.inc - out.puts(s"throw new ConsistencyError($msgStr, $actual, $expected);") + out.puts(s"throw new ConsistencyError($msgStr, ${expression(actual)}, ${expression(expected)});") out.dec importList.add("io.kaitai.struct.ConsistencyError") diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala index 396da4b4a..9a36de286 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala @@ -94,6 +94,7 @@ trait EveryWriteIsExpression extends LanguageCompiler with ObjectOrientedLanguag val expr = writeExprAsExpr(idToWrite, rep, isRaw) attrPrimitiveWrite(io, expr, t, None) t.terminator.foreach { (term) => + // FIXME: does not take `eos-error: false` into account (assumes `eos-error: true`) if (!t.include) attrPrimitiveWrite(io, Ast.expr.IntNum(term), Int1Type(false), None) } @@ -108,6 +109,7 @@ trait EveryWriteIsExpression extends LanguageCompiler with ObjectOrientedLanguag blockScopeHeader pushPos(io) } + // FIXME: does not take `eos-error: false` into account (assumes `eos-error: true`) attrPrimitiveWrite(io, Ast.expr.IntNum(t.terminator), Int1Type(false), None) if (!t.consume) { popPos(io) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/GenericChecks.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/GenericChecks.scala index cb73b9324..8b69c7cd6 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/GenericChecks.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/GenericChecks.scala @@ -35,13 +35,12 @@ trait GenericChecks extends LanguageCompiler with EveryReadIsExpression with Eve val item = writeExprAsExpr(id, repeat, isRaw) dataType match { case t: BytesType => - attrByteSizeCheck(item, t, exprByteArraySize(item), idToMsg(id)) + attrBytesCheck(item, t, idToMsg(id)) case st: StrFromBytesType => val bytes = exprStrToBytes(item, st.encoding) - attrByteSizeCheck( + attrBytesCheck( bytes, st.bytes, - exprByteArraySize(bytes), idToMsg(id) ) case _ => // no checks @@ -55,45 +54,111 @@ trait GenericChecks extends LanguageCompiler with EveryReadIsExpression with Eve idToMsg(id) ) - def attrByteSizeCheck(name: Ast.expr, t: BytesType, actualSize: Ast.expr, msgId: String): Unit = { + def attrBytesCheck(bytes: Ast.expr, t: BytesType, msgId: String): Unit = { + val actualSize = exprByteArraySize(bytes) t match { - case blt: BytesLimitType => - if (blt.padRight.isDefined) { - // size must be "<= declared" + case blt: BytesLimitType => { + if (blt.terminator.isDefined || blt.padRight.isDefined) { + // size must be "<= declared" (less than or equal to declared size) attrAssertLtE(actualSize, blt.size, msgId) - blt.terminator.foreach { (term) => - if (blt.include) - attrAssertLastByte(name, term, msgId) - } } else { - blt.terminator match { - case Some(term) => - if (!blt.include) { - // size must be "<= (declared - 1)", i.e. "< declared" - attrAssertLt(actualSize, blt.size, msgId) - } else { - // terminator is included into the string, so - // size must be "<= declared" - attrAssertLtE(actualSize, blt.size, msgId) - attrAssertLastByte(name, term, msgId) - } - case None => - // size must match declared size exactly - attrAssertEqual( - actualSize, - blt.size, - msgId - ) + // size must match declared size exactly + attrAssertEqual( + actualSize, + blt.size, + msgId + ) + } + blt.terminator.foreach { (term) => + val actualIndexOfTerm = exprByteArrayIndexOf(bytes, term) + val isPadRightActive = blt.padRight.map(padByte => padByte != term).getOrElse(false) + if (!blt.include) { + attrAssertEqual(actualIndexOfTerm, Ast.expr.IntNum(-1), msgId) + if (isPadRightActive) { + condIfHeader(Ast.expr.Compare(actualSize, Ast.cmpop.Eq, blt.size)) + // check if the last byte is not `pad-right` + } + } else { + val lastByteIndex = Ast.expr.BinOp(actualSize, Ast.operator.Sub, Ast.expr.IntNum(1)) + if (!isPadRightActive) { + condIfHeader(Ast.expr.Compare(actualSize, Ast.cmpop.Lt, blt.size)) + // must not be empty (always contains at least the `terminator` byte) + attrAssertCmp(actualSize, Ast.cmpop.Eq, Ast.expr.IntNum(0), msgId) + // the user wants to terminate the value prematurely and there's no `pad-right` that + // could do that, so the last byte of the value must be `terminator` + attrAssertEqual(actualIndexOfTerm, lastByteIndex, msgId) + condIfFooter + + condIfHeader(Ast.expr.Compare(actualSize, Ast.cmpop.Eq, blt.size)) + } + attrBasicCheck( + Ast.expr.BoolOp( + Ast.boolop.And, + Seq( + Ast.expr.Compare(actualIndexOfTerm, Ast.cmpop.NotEq, Ast.expr.IntNum(-1)), + Ast.expr.Compare(actualIndexOfTerm, Ast.cmpop.NotEq, lastByteIndex) + ) + ), + actualIndexOfTerm, + lastByteIndex, + msgId + ) + if (!isPadRightActive) { + condIfFooter + } else { + condIfHeader(Ast.expr.Compare(actualIndexOfTerm, Ast.cmpop.Eq, Ast.expr.IntNum(-1))) + // check if the last byte is not `pad-right` + } } + // intentionally deferring the `condIfFooter` call (in case of `isPadRightActive`) + } + blt.padRight.foreach { (padByte) => + if (blt.terminator.map(term => padByte != term).getOrElse(true)) { + val lastByte = exprByteArrayLastByte(bytes) + attrBasicCheck( + Ast.expr.BoolOp( + Ast.boolop.And, + Seq( + Ast.expr.Compare(actualSize, Ast.cmpop.NotEq, Ast.expr.IntNum(0)), + Ast.expr.Compare( + lastByte, + Ast.cmpop.Eq, + Ast.expr.IntNum(padByte) + ) + ) + ), + lastByte, + Ast.expr.IntNum(padByte), + msgId + ) + } + } + blt.terminator.foreach { (term) => + val isPadRightActive = blt.padRight.map(padByte => padByte != term).getOrElse(false) + // here's the `condIfFooter` call omitted from the previous `blt.terminator.foreach()` block + if (isPadRightActive) + condIfFooter } - case btt: BytesTerminatedType => - if (btt.include) - attrAssertLastByte(name, btt.terminator, msgId) + } + case btt: BytesTerminatedType => { + val actualIndexOfTerm = exprByteArrayIndexOf(bytes, btt.terminator) + // FIXME: does not take `eos-error: false` into account (assumes `eos-error: true`, i.e. the default setting) + val expectedIndexOfTerm = if (btt.include) { + // must not be empty (always contains at least the `terminator` byte) + attrAssertCmp(actualSize, Ast.cmpop.Eq, Ast.expr.IntNum(0), msgId) + + Ast.expr.BinOp(actualSize, Ast.operator.Sub, Ast.expr.IntNum(1)) + } else { + Ast.expr.IntNum(-1) + } + + attrAssertEqual(actualIndexOfTerm, expectedIndexOfTerm, msgId) + } case _ => // no checks } } - def attrBasicCheck(checkExpr: String, actual: String, expected: String, msg: String): Unit + def attrBasicCheck(checkExpr: Ast.expr, actual: Ast.expr, expected: Ast.expr, msg: String): Unit private def idToMsg(id: Identifier): String = id match { @@ -111,18 +176,22 @@ trait GenericChecks extends LanguageCompiler with EveryReadIsExpression with Eve Ast.identifier("size") ) - def exprArraySize(name: Ast.expr) = exprByteArraySize(name) + def exprByteArrayLastByte(name: Ast.expr) = + Ast.expr.Attribute( + name, + Ast.identifier("last") + ) - def attrAssertLastByte(name: Ast.expr, expectedLast: Int, msg: String): Unit = { - attrAssertEqual( + def exprByteArrayIndexOf(name: Ast.expr, term: Int) = + Ast.expr.Call( Ast.expr.Attribute( name, - Ast.identifier("last") + Ast.identifier("index_of") ), - Ast.expr.IntNum(expectedLast), - msg + Seq(Ast.expr.IntNum(term)) ) - } + + def exprArraySize(name: Ast.expr) = exprByteArraySize(name) def attrAssertEqual(actual: Ast.expr, expected: Ast.expr, msg: String): Unit = attrAssertCmp(actual, Ast.cmpop.NotEq, expected, msg) @@ -130,14 +199,11 @@ trait GenericChecks extends LanguageCompiler with EveryReadIsExpression with Eve def attrAssertLtE(actual: Ast.expr, expected: Ast.expr, msg: String): Unit = attrAssertCmp(actual, Ast.cmpop.Gt, expected, msg) - def attrAssertLt(actual: Ast.expr, expected: Ast.expr, msg: String): Unit = - attrAssertCmp(actual, Ast.cmpop.GtE, expected, msg) - def attrAssertCmp(actual: Ast.expr, op: Ast.cmpop, expected: Ast.expr, msg: String): Unit = attrBasicCheck( - translator.translate(Ast.expr.Compare(actual, op, expected)), - translator.translate(actual), - translator.translate(expected), + Ast.expr.Compare(actual, op, expected), + actual, + expected, msg ) } diff --git a/shared/src/main/scala/io/kaitai/struct/translators/CommonMethods.scala b/shared/src/main/scala/io/kaitai/struct/translators/CommonMethods.scala index adeb077c4..6711e620c 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/CommonMethods.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/CommonMethods.scala @@ -99,6 +99,7 @@ abstract trait CommonMethods[T] extends TypeDetector { case (_: StrType, "to_i") => strToInt(obj, args(0)) case (_: StrType, "to_b") => strToBytes(obj, args(0)) case (_: BytesType, "to_s") => bytesToStr(obj, args(0)) + case (_: BytesType, "index_of") => bytesIndexOf(obj, args(0)) case _ => throw new TypeMismatchError(s"don't know how to call method '$methodName' of object type '$objType'") } } @@ -121,6 +122,7 @@ abstract trait CommonMethods[T] extends TypeDetector { def strToBytes(s: Ast.expr, encoding: Ast.expr): T = ??? def bytesToStr(value: Ast.expr, expr: Ast.expr): T + def bytesIndexOf(value: Ast.expr, expr: Ast.expr): T = ??? def intToStr(value: Ast.expr, num: Ast.expr): T diff --git a/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala index 8a7c44c5f..2a7cbdab3 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala @@ -125,6 +125,8 @@ class JavaTranslator(provider: TypeProvider, importList: ImportList, config: Run importList.add("java.nio.charset.Charset") s"new String($bytesExpr, Charset.forName(${translate(encoding)}))" } + override def bytesIndexOf(b: expr, byte: expr): String = + s"${JavaCompiler.kstreamName}.byteArrayIndexOf(${translate(b)}, (byte) ${translate(byte)})" override def bytesLength(b: Ast.expr): String = s"${translate(b)}.length" diff --git a/shared/src/main/scala/io/kaitai/struct/translators/TypeDetector.scala b/shared/src/main/scala/io/kaitai/struct/translators/TypeDetector.scala index 510bb2faf..89c5e4d38 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/TypeDetector.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/TypeDetector.scala @@ -231,6 +231,7 @@ class TypeDetector(provider: TypeProvider) { case (_: StrType, "to_i") => CalcIntType case (_: StrType, "to_b") => CalcBytesType case (_: BytesType, "to_s") => CalcStrType + case (_: BytesType, "index_of") => CalcIntType case _ => throw new MethodNotFoundError(methodName.name, objType) } From d721d08b07b342b850646dc07b973fa314e86be4 Mon Sep 17 00:00:00 2001 From: Petr Pucil Date: Sun, 11 Sep 2022 12:24:06 +0200 Subject: [PATCH 41/90] Reuse the byte array write method for strings --- .../components/EveryWriteIsExpression.scala | 39 ++++++------------- 1 file changed, 11 insertions(+), 28 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala index 9a36de286..6dcf81c07 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala @@ -80,7 +80,7 @@ trait EveryWriteIsExpression extends LanguageCompiler with ObjectOrientedLanguag } } - def attrBytesTypeWrite(id: Identifier, t: BytesType, io: String, rep: RepeatSpec, isRaw: Boolean) = { + def attrBytesTypeWrite(id: Identifier, t: BytesType, io: String, rep: RepeatSpec, isRaw: Boolean): Unit = { val idToWrite = t.process match { case Some(proc) => val rawId = RawIdentifier(id) @@ -89,9 +89,18 @@ trait EveryWriteIsExpression extends LanguageCompiler with ObjectOrientedLanguag case None => id } + val expr = writeExprAsExpr(idToWrite, rep, isRaw) + attrBytesTypeWrite2(io, expr, t) + } + + def attrStrTypeWrite(id: Identifier, t: StrFromBytesType, io: String, rep: RepeatSpec, isRaw: Boolean): Unit = { + val expr = exprStrToBytes(writeExprAsExpr(id, rep, isRaw), t.encoding) + attrBytesTypeWrite2(io, expr, t.bytes) + } + + def attrBytesTypeWrite2(io: String, expr: Ast.expr, t: BytesType): Unit = t match { case t: BytesEosType => - val expr = writeExprAsExpr(idToWrite, rep, isRaw) attrPrimitiveWrite(io, expr, t, None) t.terminator.foreach { (term) => // FIXME: does not take `eos-error: false` into account (assumes `eos-error: true`) @@ -99,10 +108,8 @@ trait EveryWriteIsExpression extends LanguageCompiler with ObjectOrientedLanguag attrPrimitiveWrite(io, Ast.expr.IntNum(term), Int1Type(false), None) } case blt: BytesLimitType => - val expr = writeExprAsExpr(idToWrite, rep, isRaw) attrBytesLimitWrite2(io, expr, blt) case t: BytesTerminatedType => - val expr = writeExprAsExpr(idToWrite, rep, isRaw) attrPrimitiveWrite(io, expr, t, None) if (!t.include) { if (!t.consume) { @@ -117,30 +124,6 @@ trait EveryWriteIsExpression extends LanguageCompiler with ObjectOrientedLanguag } } } - } - - def attrStrTypeWrite(id: Identifier, t: StrFromBytesType, io: String, rep: RepeatSpec, isRaw: Boolean) = { - val expr = exprStrToBytes(writeExprAsExpr(id, rep, isRaw), t.encoding) - attrPrimitiveWrite(io, expr, t.bytes, None) - - /** FIXME: duplication with the previous [[attrBytesTypeWrite]] method (will also ensure consistent handling) */ - t.bytes match { - case t: BytesEosType => - t.terminator.foreach { (term) => - if (!t.include) - attrPrimitiveWrite(io, Ast.expr.IntNum(term), Int1Type(false), None) - } - case t: BytesLimitType => - // FIXME: implement padding and terminator byte - t.terminator.foreach { (term) => - if (!t.include) - attrPrimitiveWrite(io, Ast.expr.IntNum(term), Int1Type(false), None) - } - case t: BytesTerminatedType => - if (t.consume && !t.include) - attrPrimitiveWrite(io, Ast.expr.IntNum(t.terminator), Int1Type(false), None) - } - } def attrBytesLimitWrite2(io: String, expr: Ast.expr, blt: BytesLimitType): Unit = { val (term, padRight) = (blt.terminator, blt.padRight, blt.include) match { From 50d655637435d01148b0057a0bec10e9cf145af0 Mon Sep 17 00:00:00 2001 From: Petr Pucil Date: Mon, 12 Sep 2022 15:43:45 +0200 Subject: [PATCH 42/90] Add repeat-until last element check I reuse ConsistencyError for throwing if the last element of array doesn't satisfy repeat-until condition, but I'm not sure what should be present in the error message. Maybe it should throw a special type of Exception deriving from ConsistencyError, but I don't know. --- .../languages/components/GenericChecks.scala | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/GenericChecks.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/GenericChecks.scala index 8b69c7cd6..42de81671 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/GenericChecks.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/GenericChecks.scala @@ -21,6 +21,7 @@ trait GenericChecks extends LanguageCompiler with EveryReadIsExpression with Eve attrCheck2(id, attr.dataType, io, attr.cond.repeat, false) condRepeatCommonFooter case RepeatUntil(untilExpr: Ast.expr) => + attrAssertUntilCond(Ast.expr.InternalName(id), attr.dataType, untilExpr, idToMsg(id)) condRepeatCommonHeader(id, io, attr.dataType) attrCheck2(id, attr.dataType, io, attr.cond.repeat, false) condRepeatCommonFooter @@ -193,6 +194,28 @@ trait GenericChecks extends LanguageCompiler with EveryReadIsExpression with Eve def exprArraySize(name: Ast.expr) = exprByteArraySize(name) + def attrAssertUntilCond(name: Ast.expr, dataType: DataType, untilExpr: Ast.expr, msg: String): Unit = { + blockScopeHeader + handleAssignmentTempVar( + dataType, + translator.doName(Identifier.ITERATOR), + translator.translate( + Ast.expr.Attribute( + name, + Ast.identifier("last") + ) + ) + ) + typeProvider._currentIteratorType = Some(dataType) + attrBasicCheck( + Ast.expr.UnaryOp(Ast.unaryop.Not, untilExpr), + Ast.expr.Str(dataType.toString), + Ast.expr.Str(translator.translate(untilExpr)), + msg + ) + blockScopeFooter + } + def attrAssertEqual(actual: Ast.expr, expected: Ast.expr, msg: String): Unit = attrAssertCmp(actual, Ast.cmpop.NotEq, expected, msg) From 72ca925266f4b4b06c8384bfadbdba20f08455a2 Mon Sep 17 00:00:00 2001 From: Petr Pucil Date: Mon, 12 Sep 2022 16:00:33 +0200 Subject: [PATCH 43/90] Refactor access to the last array item in attrAssertUntilCond --- .../struct/languages/components/GenericChecks.scala | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/GenericChecks.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/GenericChecks.scala index 42de81671..b703e8006 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/GenericChecks.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/GenericChecks.scala @@ -115,7 +115,7 @@ trait GenericChecks extends LanguageCompiler with EveryReadIsExpression with Eve } blt.padRight.foreach { (padByte) => if (blt.terminator.map(term => padByte != term).getOrElse(true)) { - val lastByte = exprByteArrayLastByte(bytes) + val lastByte = exprByteArrayLast(bytes) attrBasicCheck( Ast.expr.BoolOp( Ast.boolop.And, @@ -177,7 +177,7 @@ trait GenericChecks extends LanguageCompiler with EveryReadIsExpression with Eve Ast.identifier("size") ) - def exprByteArrayLastByte(name: Ast.expr) = + def exprByteArrayLast(name: Ast.expr) = Ast.expr.Attribute( name, Ast.identifier("last") @@ -194,17 +194,14 @@ trait GenericChecks extends LanguageCompiler with EveryReadIsExpression with Eve def exprArraySize(name: Ast.expr) = exprByteArraySize(name) + def exprArrayLast(name: Ast.expr) = exprByteArrayLast(name) + def attrAssertUntilCond(name: Ast.expr, dataType: DataType, untilExpr: Ast.expr, msg: String): Unit = { blockScopeHeader handleAssignmentTempVar( dataType, translator.doName(Identifier.ITERATOR), - translator.translate( - Ast.expr.Attribute( - name, - Ast.identifier("last") - ) - ) + translator.translate(exprArrayLast(name)) ) typeProvider._currentIteratorType = Some(dataType) attrBasicCheck( From def665036159fbb23e8f7ebac04c0ba06a3039c3 Mon Sep 17 00:00:00 2001 From: Petr Pucil Date: Mon, 12 Sep 2022 19:51:36 +0200 Subject: [PATCH 44/90] Extend repeat-until consistency check to all elements See https://github.com/kaitai-io/kaitai_struct_compiler/pull/183#issuecomment-1066722278 --- .../languages/components/GenericChecks.scala | 19 +++++++++++-------- .../struct/translators/JavaTranslator.scala | 3 +++ 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/GenericChecks.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/GenericChecks.scala index b703e8006..265fe9447 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/GenericChecks.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/GenericChecks.scala @@ -20,10 +20,10 @@ trait GenericChecks extends LanguageCompiler with EveryReadIsExpression with Eve condRepeatCommonHeader(id, io, attr.dataType) attrCheck2(id, attr.dataType, io, attr.cond.repeat, false) condRepeatCommonFooter - case RepeatUntil(untilExpr: Ast.expr) => - attrAssertUntilCond(Ast.expr.InternalName(id), attr.dataType, untilExpr, idToMsg(id)) + case rep: RepeatUntil => condRepeatCommonHeader(id, io, attr.dataType) attrCheck2(id, attr.dataType, io, attr.cond.repeat, false) + attrAssertUntilCond(id, attr.dataType, rep, false, idToMsg(id)) condRepeatCommonFooter case NoRepeat => attrCheck2(id, attr.dataType, io, attr.cond.repeat, false) @@ -196,18 +196,21 @@ trait GenericChecks extends LanguageCompiler with EveryReadIsExpression with Eve def exprArrayLast(name: Ast.expr) = exprByteArrayLast(name) - def attrAssertUntilCond(name: Ast.expr, dataType: DataType, untilExpr: Ast.expr, msg: String): Unit = { + def attrAssertUntilCond(id: Identifier, dataType: DataType, repeat: RepeatUntil, isRaw: Boolean, msg: String): Unit = { blockScopeHeader handleAssignmentTempVar( dataType, translator.doName(Identifier.ITERATOR), - translator.translate(exprArrayLast(name)) + translator.translate(writeExprAsExpr(id, repeat, isRaw)) ) typeProvider._currentIteratorType = Some(dataType) - attrBasicCheck( - Ast.expr.UnaryOp(Ast.unaryop.Not, untilExpr), - Ast.expr.Str(dataType.toString), - Ast.expr.Str(translator.translate(untilExpr)), + attrAssertEqual( + repeat.expr, + Ast.expr.Compare( + Ast.expr.Name(Ast.identifier(Identifier.INDEX)), + Ast.cmpop.Eq, + Ast.expr.BinOp(exprArraySize(Ast.expr.InternalName(id)), Ast.operator.Sub, Ast.expr.IntNum(1)) + ), msg ) blockScopeFooter diff --git a/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala index 2a7cbdab3..ff99ef3be 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala @@ -57,6 +57,9 @@ class JavaTranslator(provider: TypeProvider, importList: ImportList, config: Run } } + override def doNumericCompareOp(left: expr, op: cmpop, right: expr): String = + s"(${super.doNumericCompareOp(left, op, right)})" + override def doName(s: String) = s match { case Identifier.ITERATOR => "_it" From 306bfb2738c7a56fa97a8491963bc80b3350ee3e Mon Sep 17 00:00:00 2001 From: Petr Pucil Date: Mon, 12 Sep 2022 20:52:38 +0200 Subject: [PATCH 45/90] Add consistency check for *non-empty* `repeat: until` array `repeat: until` is very similar to `terminator` + `include: true`, which also uses this check. --- .../io/kaitai/struct/languages/components/GenericChecks.scala | 2 ++ 1 file changed, 2 insertions(+) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/GenericChecks.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/GenericChecks.scala index 265fe9447..7a77b4adc 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/GenericChecks.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/GenericChecks.scala @@ -21,6 +21,8 @@ trait GenericChecks extends LanguageCompiler with EveryReadIsExpression with Eve attrCheck2(id, attr.dataType, io, attr.cond.repeat, false) condRepeatCommonFooter case rep: RepeatUntil => + // the array must not be empty (always contains at least the `repeat-until: {true}` element) + attrAssertCmp(exprArraySize(Ast.expr.InternalName(id)), Ast.cmpop.Eq, Ast.expr.IntNum(0), idToMsg(id)) condRepeatCommonHeader(id, io, attr.dataType) attrCheck2(id, attr.dataType, io, attr.cond.repeat, false) attrAssertUntilCond(id, attr.dataType, rep, false, idToMsg(id)) From 849fc1c595837aa4dfb9f340842a60e5e097f98f Mon Sep 17 00:00:00 2001 From: Petr Pucil Date: Tue, 13 Sep 2022 16:22:24 +0200 Subject: [PATCH 46/90] Fix "Object cannot be converted to byte[]" errors in type switches --- .../struct/languages/JavaCompiler.scala | 12 +++++-- .../components/EveryWriteIsExpression.scala | 36 +++++++++---------- 2 files changed, 27 insertions(+), 21 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala index f4beb23ff..aae1ec613 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala @@ -315,8 +315,14 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } // TODO: merge with attrProcess above (there is currently 99.9% duplication) - override def attrUnprocess(proc: ProcessExpr, varSrc: Identifier, varDest: Identifier, rep: RepeatSpec): Unit = { - val srcExpr = getRawIdExpr(varSrc, rep) + override def attrUnprocess(proc: ProcessExpr, varSrc: Identifier, varDest: Identifier, rep: RepeatSpec, dataType: BytesType, exprTypeOpt: Option[DataType]): Unit = { + val exprType = exprTypeOpt.getOrElse(dataType) + val srcExprRaw = getRawIdExpr(varSrc, rep) + val srcExpr = if (exprType != dataType) { + s"(${kaitaiType2JavaType(dataType)}) ($srcExprRaw)" + } else { + srcExprRaw + } val expr = proc match { case ProcessXor(xorValue) => @@ -824,7 +830,7 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) valueExpr: Ast.expr, dataType: DataType, defEndian: Option[FixedEndian], - exprTypeOpt: Option[DataType] = None + exprTypeOpt: Option[DataType] ): Unit = { val exprType = exprTypeOpt.getOrElse(dataType) val exprRaw = expression(valueExpr) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala index 6dcf81c07..6977373c4 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala @@ -46,7 +46,7 @@ trait EveryWriteIsExpression extends LanguageCompiler with ObjectOrientedLanguag case t: UserType => attrUserTypeWrite(id, t, io, rep, isRaw, defEndian, exprTypeOpt) case t: BytesType => - attrBytesTypeWrite(id, t, io, rep, isRaw) + attrBytesTypeWrite(id, t, io, rep, isRaw, exprTypeOpt) case st: SwitchType => val isNullable = if (switchBytesOnlyAsRaw) { st.isNullableSwitchRaw @@ -56,7 +56,7 @@ trait EveryWriteIsExpression extends LanguageCompiler with ObjectOrientedLanguag attrSwitchTypeWrite(id, st.on, st.cases, io, rep, defEndian, isNullable, st.combinedType) case t: StrFromBytesType => - attrStrTypeWrite(id, t, io, rep, isRaw) + attrStrTypeWrite(id, t, io, rep, isRaw, exprTypeOpt) case t: EnumType => val expr = writeExprAsExpr(id, rep, isRaw) val exprType = internalEnumIntType(t.basedOn) @@ -80,44 +80,44 @@ trait EveryWriteIsExpression extends LanguageCompiler with ObjectOrientedLanguag } } - def attrBytesTypeWrite(id: Identifier, t: BytesType, io: String, rep: RepeatSpec, isRaw: Boolean): Unit = { + def attrBytesTypeWrite(id: Identifier, t: BytesType, io: String, rep: RepeatSpec, isRaw: Boolean, exprTypeOpt: Option[DataType]): Unit = { val idToWrite = t.process match { case Some(proc) => val rawId = RawIdentifier(id) - attrUnprocess(proc, id, rawId, rep) + attrUnprocess(proc, id, rawId, rep, t, exprTypeOpt) rawId case None => id } val expr = writeExprAsExpr(idToWrite, rep, isRaw) - attrBytesTypeWrite2(io, expr, t) + attrBytesTypeWrite2(io, expr, t, exprTypeOpt) } - def attrStrTypeWrite(id: Identifier, t: StrFromBytesType, io: String, rep: RepeatSpec, isRaw: Boolean): Unit = { + def attrStrTypeWrite(id: Identifier, t: StrFromBytesType, io: String, rep: RepeatSpec, isRaw: Boolean, exprTypeOpt: Option[DataType]): Unit = { val expr = exprStrToBytes(writeExprAsExpr(id, rep, isRaw), t.encoding) - attrBytesTypeWrite2(io, expr, t.bytes) + attrBytesTypeWrite2(io, expr, t.bytes, exprTypeOpt) } - def attrBytesTypeWrite2(io: String, expr: Ast.expr, t: BytesType): Unit = + def attrBytesTypeWrite2(io: String, expr: Ast.expr, t: BytesType, exprTypeOpt: Option[DataType]): Unit = t match { case t: BytesEosType => - attrPrimitiveWrite(io, expr, t, None) + attrPrimitiveWrite(io, expr, t, None, exprTypeOpt) t.terminator.foreach { (term) => // FIXME: does not take `eos-error: false` into account (assumes `eos-error: true`) if (!t.include) - attrPrimitiveWrite(io, Ast.expr.IntNum(term), Int1Type(false), None) + attrPrimitiveWrite(io, Ast.expr.IntNum(term), Int1Type(false), None, None) } case blt: BytesLimitType => - attrBytesLimitWrite2(io, expr, blt) + attrBytesLimitWrite2(io, expr, blt, exprTypeOpt) case t: BytesTerminatedType => - attrPrimitiveWrite(io, expr, t, None) + attrPrimitiveWrite(io, expr, t, None, exprTypeOpt) if (!t.include) { if (!t.consume) { blockScopeHeader pushPos(io) } // FIXME: does not take `eos-error: false` into account (assumes `eos-error: true`) - attrPrimitiveWrite(io, Ast.expr.IntNum(t.terminator), Int1Type(false), None) + attrPrimitiveWrite(io, Ast.expr.IntNum(t.terminator), Int1Type(false), None, None) if (!t.consume) { popPos(io) blockScopeFooter @@ -125,12 +125,12 @@ trait EveryWriteIsExpression extends LanguageCompiler with ObjectOrientedLanguag } } - def attrBytesLimitWrite2(io: String, expr: Ast.expr, blt: BytesLimitType): Unit = { + def attrBytesLimitWrite2(io: String, expr: Ast.expr, blt: BytesLimitType, exprTypeOpt: Option[DataType]): Unit = { val (term, padRight) = (blt.terminator, blt.padRight, blt.include) match { case (None, None, false) => // no terminator, no padding => just a regular output // validation should check that expression's length matches size - attrPrimitiveWrite(io, expr, blt, None) + attrPrimitiveWrite(io, expr, blt, None, exprTypeOpt) return case (_, None, true) => // terminator included, no padding => pad with zeroes @@ -195,7 +195,7 @@ trait EveryWriteIsExpression extends LanguageCompiler with ObjectOrientedLanguag val ioFixed = thisLocal.allocateIOFixed(rawId, translator.translate(blt.size)) attrUserTypeInstreamWrite(ioFixed, expr, t, exprType) handleAssignment(rawId, exprStreamToByteArray(ioFixed), rep, isRaw) - attrBytesTypeWrite(rawId, byteType, io, rep, isRaw) + attrBytesTypeWrite(rawId, byteType, io, rep, isRaw, exprTypeOpt) } } } @@ -244,11 +244,11 @@ trait EveryWriteIsExpression extends LanguageCompiler with ObjectOrientedLanguag def internalEnumIntType(basedOn: IntType): DataType - def attrPrimitiveWrite(io: String, expr: Ast.expr, dt: DataType, defEndian: Option[FixedEndian], exprTypeOpt: Option[DataType] = None): Unit + def attrPrimitiveWrite(io: String, expr: Ast.expr, dt: DataType, defEndian: Option[FixedEndian], exprTypeOpt: Option[DataType]): Unit def attrBytesLimitWrite(io: String, expr: Ast.expr, size: String, term: Int, padRight: Int): Unit def attrUserTypeInstreamWrite(io: String, expr: Ast.expr, t: DataType, exprType: DataType): Unit def attrWriteStreamToStream(srcIo: String, dstIo: String): Unit def exprStreamToByteArray(ioFixed: String): String - def attrUnprocess(proc: ProcessExpr, varSrc: Identifier, varDest: Identifier, rep: RepeatSpec): Unit + def attrUnprocess(proc: ProcessExpr, varSrc: Identifier, varDest: Identifier, rep: RepeatSpec, dt: BytesType, exprTypeOpt: Option[DataType]): Unit } From 74d553cd508062346040e83ef1b6d2885af55513 Mon Sep 17 00:00:00 2001 From: Petr Pucil Date: Wed, 14 Sep 2022 13:22:13 +0200 Subject: [PATCH 47/90] Fix locale-sensitive String#toUpperCase call in writeHeader Using this function has already caused problems in the past, see https://github.com/kaitai-io/kaitai_struct/issues/708. --- .../main/scala/io/kaitai/struct/languages/JavaCompiler.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala index aae1ec613..fcf1a411e 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala @@ -226,7 +226,7 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def writeHeader(endian: Option[FixedEndian]): Unit = { val suffix = endian match { - case Some(e) => s"${e.toSuffix.toUpperCase}" + case Some(e) => Utils.upperUnderscoreCase(e.toSuffix) case None => "" } out.puts From 604359d63b7d98511828913e206a85f105d3993a Mon Sep 17 00:00:00 2001 From: Petr Pucil Date: Thu, 15 Sep 2022 23:49:48 +0200 Subject: [PATCH 48/90] Java: finally solve boxed numeric type casting issues consistently The most common error message as these problems manifest themselves is "Long cannot be converted to int". This commit addresses all identified places where errors like that can occur. Resolves https://github.com/kaitai-io/kaitai_struct/issues/956 Uses the solution from https://github.com/kaitai-io/kaitai_struct_compiler/pull/149 --- .../struct/languages/JavaCompiler.scala | 54 +++++++++---------- .../struct/translators/JavaTranslator.scala | 15 ++++-- 2 files changed, 34 insertions(+), 35 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala index fcf1a411e..3cc69b7c4 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala @@ -318,11 +318,7 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def attrUnprocess(proc: ProcessExpr, varSrc: Identifier, varDest: Identifier, rep: RepeatSpec, dataType: BytesType, exprTypeOpt: Option[DataType]): Unit = { val exprType = exprTypeOpt.getOrElse(dataType) val srcExprRaw = getRawIdExpr(varSrc, rep) - val srcExpr = if (exprType != dataType) { - s"(${kaitaiType2JavaType(dataType)}) ($srcExprRaw)" - } else { - srcExprRaw - } + val srcExpr = castIfNeeded(srcExprRaw, exprType, dataType) val expr = proc match { case ProcessXor(xorValue) => @@ -572,11 +568,7 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) s"new ${types2class(t.name)}($io$addArgs$addParams)" } - if (assignType != dataType) { - s"(${kaitaiType2JavaType(assignType)}) ($expr)" - } else { - expr - } + castIfNeeded(expr, dataType, assignType) } override def bytesPadTermExpr(expr0: String, padRight: Option[Int], terminator: Option[Int], include: Boolean) = { @@ -593,11 +585,7 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } override def userTypeDebugRead(id: String, dataType: DataType, assignType: DataType): Unit = { - val expr = if (assignType != dataType) { - s"((${kaitaiType2JavaType(dataType)}) ($id))" - } else { - id - } + val expr = castIfNeeded(id, assignType, dataType) out.puts(s"$expr._read();") } @@ -762,14 +750,13 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) val primType = kaitaiType2JavaTypePrim(dataType) val boxedType = kaitaiType2JavaTypeBoxed(dataType) - if (primType != boxedType) { - // Special trick to achieve both implicit type conversion + boxing. - // Unfortunately, Java can't do both in one assignment, i.e. this would fail: + if (dataType.isInstanceOf[NumericType]) { + // Special trick to achieve both type conversion + boxing. + // Unfortunately, Java can't do both by itself, i.e. this would fail: // // Double c = 1.0f + 1; - out.puts(s"$primType _tmp = ($primType) (${expression(value)});") - out.puts(s"${privateMemberName(instName)} = _tmp;") + out.puts(s"${privateMemberName(instName)} = ${translator.doCast(value, dataType)};") } else { out.puts(s"${privateMemberName(instName)} = ${expression(value)};") } @@ -834,11 +821,7 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) ): Unit = { val exprType = exprTypeOpt.getOrElse(dataType) val exprRaw = expression(valueExpr) - val expr = if (exprType != dataType) { - s"(${kaitaiType2JavaType(dataType)}) ($exprRaw)" - } else { - exprRaw - } + val expr = castIfNeeded(exprRaw, exprType, dataType) val stmt = dataType match { case t: ReadableType => @@ -858,11 +841,7 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def attrUserTypeInstreamWrite(io: String, valueExpr: Ast.expr, dataType: DataType, exprType: DataType) = { val exprRaw = expression(valueExpr) - val expr = if (exprType != dataType) { - s"((${kaitaiType2JavaType(dataType)}) ($exprRaw))" - } else { - exprRaw - } + val expr = castIfNeeded(exprRaw, exprType, dataType) out.puts(s"$expr._write($io);") } @@ -940,6 +919,21 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) kaitaiType2JavaTypePrim(attrType) } + def castIfNeeded(exprRaw: String, exprType: DataType, targetType: DataType): String = + if (exprType != targetType) { + val castTypeId = kaitaiType2JavaTypePrim(targetType) + targetType match { + // Handles both unboxing + downcasting at the same time if needed + // (solution from https://github.com/kaitai-io/kaitai_struct_compiler/pull/149) + // + // See also https://github.com/kaitai-io/kaitai_struct_compiler/pull/212#issuecomment-731149487 + case _: NumericType => s"((Number) ($exprRaw)).${castTypeId}Value()" + case _ => s"(($castTypeId) ($exprRaw))" + } + } else { + exprRaw + } + /** * Determine Java data type corresponding to a KS data type. A "primitive" type (i.e. "int", "long", etc) will * be returned if possible. diff --git a/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala index ff99ef3be..b5563b786 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala @@ -106,13 +106,18 @@ class JavaTranslator(provider: TypeProvider, importList: ImportList, config: Run } override def arraySubscript(container: expr, idx: expr): String = - s"${translate(container)}.get((int) ${translate(idx)})" + s"${translate(container)}.get(${doCast(idx, CalcIntType)})" override def doIfExp(condition: expr, ifTrue: expr, ifFalse: expr): String = s"(${translate(condition)} ? ${translate(ifTrue)} : ${translate(ifFalse)})" override def doCast(value: Ast.expr, typeName: DataType): String = { // FIXME val compiler = new JavaCompiler(provider.asInstanceOf[ClassTypeProvider], config) - s"((${compiler.kaitaiType2JavaType(typeName)}) (${translate(value)}))" + if (value.isInstanceOf[Ast.expr.IntNum] || value.isInstanceOf[Ast.expr.FloatNum]) + // this branch is not really needed, but makes the code a bit cleaner - + // we can simplify casting to just this for numeric constants + s"((${compiler.kaitaiType2JavaType(typeName)}) ${translate(value)})" + else + compiler.castIfNeeded(translate(value), AnyType, typeName) } // Predefined methods of various types @@ -121,7 +126,7 @@ class JavaTranslator(provider: TypeProvider, importList: ImportList, config: Run override def enumToInt(v: expr, et: EnumType): String = s"${translate(v)}.id()" override def floatToInt(v: expr): String = - s"(int) (${translate(v)} + 0)" + doCast(v, CalcIntType) override def intToStr(i: expr, base: expr): String = s"Long.toString(${translate(i)}, ${translate(base)})" override def bytesToStr(bytesExpr: String, encoding: Ast.expr): String = { @@ -129,12 +134,12 @@ class JavaTranslator(provider: TypeProvider, importList: ImportList, config: Run s"new String($bytesExpr, Charset.forName(${translate(encoding)}))" } override def bytesIndexOf(b: expr, byte: expr): String = - s"${JavaCompiler.kstreamName}.byteArrayIndexOf(${translate(b)}, (byte) ${translate(byte)})" + s"${JavaCompiler.kstreamName}.byteArrayIndexOf(${translate(b)}, ${doCast(byte, Int1Type(true))})" override def bytesLength(b: Ast.expr): String = s"${translate(b)}.length" override def bytesSubscript(container: Ast.expr, idx: Ast.expr): String = - s"(${translate(container)}[${translate(idx)}] & 0xff)" + s"(${translate(container)}[${doCast(idx, CalcIntType)}] & 0xff)" override def bytesFirst(b: Ast.expr): String = bytesSubscript(b, Ast.expr.IntNum(0)) override def bytesLast(b: Ast.expr): String = From f8bb57713d746c9eff4c8bc4a4dd69559539bb60 Mon Sep 17 00:00:00 2001 From: Petr Pucil Date: Sat, 17 Sep 2022 18:25:55 +0200 Subject: [PATCH 49/90] Add _invalidate*() for value and set*() for parse instances --- .../main/scala/io/kaitai/struct/ClassCompiler.scala | 11 +++++++++++ .../io/kaitai/struct/languages/JavaCompiler.scala | 13 ++++++++++--- .../languages/components/LanguageCompiler.scala | 2 ++ 3 files changed, 23 insertions(+), 3 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/ClassCompiler.scala b/shared/src/main/scala/io/kaitai/struct/ClassCompiler.scala index 4bbedea59..5bb512e1a 100644 --- a/shared/src/main/scala/io/kaitai/struct/ClassCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/ClassCompiler.scala @@ -222,6 +222,9 @@ class ClassCompiler( attr.isNullable } lang.attributeReader(attr.id, attr.dataTypeComposite, isNullable) + if (config.readWrite) { + lang.attributeSetter(attr.id, attr.dataTypeComposite, isNullable) + } } } @@ -391,6 +394,14 @@ class ClassCompiler( lang.instanceReturn(instName, dataType) lang.instanceFooter + + if (config.readWrite) + instSpec match { + case pi: ParseInstanceSpec => + lang.attributeSetter(instName, dataType, instSpec.isNullable) + case _: ValueInstanceSpec => + lang.instanceInvalidate(instName) + } } def compileInstanceDeclaration(instName: InstanceIdentifier, instSpec: InstanceSpec): Unit = { diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala index 3cc69b7c4..842ffdd75 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala @@ -249,10 +249,13 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) val name = idToStr(attrName) out.puts(s"public $javaType $name() { return $name; }") + } - if (config.readWrite) { - out.puts(s"public void set${idToSetterStr(attrName)}($javaType _v) { $name = _v; }") - } + override def attributeSetter(attrName: Identifier, attrType: DataType, isNullable: Boolean): Unit = { + val javaType = kaitaiType2JavaType(attrType, isNullable) + val name = idToStr(attrName) + + out.puts(s"public void set${idToSetterStr(attrName)}($javaType _v) { $name = _v; }") } override def universalDoc(doc: DocSpec): Unit = { @@ -762,6 +765,10 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } } + override def instanceInvalidate(instName: InstanceIdentifier): Unit = { + out.puts(s"public void _invalidate${idToSetterStr(instName)}() { ${privateMemberName(instName)} = null; }") + } + override def enumDeclaration(curClass: String, enumName: String, enumColl: Seq[(Long, String)]): Unit = { val enumClass = type2class(enumName) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala index 8f6178919..16189e8ed 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala @@ -88,6 +88,7 @@ abstract class LanguageCompiler( def attributeDeclaration(attrName: Identifier, attrType: DataType, isNullable: Boolean): Unit def attributeReader(attrName: Identifier, attrType: DataType, isNullable: Boolean): Unit + def attributeSetter(attrName: Identifier, attrType: DataType, isNullable: Boolean): Unit = ??? def attributeDoc(id: Identifier, doc: DocSpec): Unit = {} def attrParse(attr: AttrLikeSpec, id: Identifier, defEndian: Option[Endianness]): Unit @@ -141,6 +142,7 @@ abstract class LanguageCompiler( def instanceCheckCacheAndReturn(instName: InstanceIdentifier, dataType: DataType): Unit def instanceReturn(instName: InstanceIdentifier, attrType: DataType): Unit def instanceCalculate(instName: Identifier, dataType: DataType, value: Ast.expr) + def instanceInvalidate(instName: InstanceIdentifier): Unit = ??? def enumDeclaration(curClass: List[String], enumName: String, enumColl: Seq[(Long, EnumValueSpec)]): Unit From 16374cc4508d662ba222095db3f75d03eb266164 Mon Sep 17 00:00:00 2001 From: Petr Pucil Date: Sat, 17 Sep 2022 18:27:30 +0200 Subject: [PATCH 50/90] Add _write*() and _check*() for parse instances --- .../io/kaitai/struct/ClassCompiler.scala | 7 +++ .../struct/languages/JavaCompiler.scala | 12 +++++ .../components/EveryWriteIsExpression.scala | 48 +++++++++++++++++-- .../components/LanguageCompiler.scala | 6 ++- .../components/UniversalFooter.scala | 2 + 5 files changed, 70 insertions(+), 5 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/ClassCompiler.scala b/shared/src/main/scala/io/kaitai/struct/ClassCompiler.scala index 5bb512e1a..e72e11271 100644 --- a/shared/src/main/scala/io/kaitai/struct/ClassCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/ClassCompiler.scala @@ -399,6 +399,13 @@ class ClassCompiler( instSpec match { case pi: ParseInstanceSpec => lang.attributeSetter(instName, dataType, instSpec.isNullable) + lang.writeInstanceHeader(instName) + lang.attrWrite(pi, instName, endian) + lang.writeInstanceFooter + + lang.checkInstanceHeader(instName) + lang.attrCheck(pi, instName) + lang.checkInstanceFooter case _: ValueInstanceSpec => lang.instanceInvalidate(instName) } diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala index 842ffdd75..cec79bc15 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala @@ -240,6 +240,18 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.inc } + override def writeInstanceHeader(instName: InstanceIdentifier): Unit = { + out.puts + out.puts(s"public void _write${idToSetterStr(instName)}() {") + out.inc + } + + override def checkInstanceHeader(instName: InstanceIdentifier): Unit = { + out.puts + out.puts(s"public void _check${idToSetterStr(instName)}() {") + out.inc + } + override def attributeDeclaration(attrName: Identifier, attrType: DataType, isNullable: Boolean): Unit = { out.puts(s"private ${kaitaiType2JavaType(attrType, isNullable)} ${idToStr(attrName)};") } diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala index 6977373c4..e88830ff9 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala @@ -7,13 +7,54 @@ import io.kaitai.struct.exprlang.Ast import io.kaitai.struct.format._ import scala.collection.mutable.ListBuffer +import io.kaitai.struct.datatype._ trait EveryWriteIsExpression extends LanguageCompiler with ObjectOrientedLanguage with EveryReadIsExpression { - override def attrWrite(attr: AttrLikeSpec, id: Identifier, defEndian: Option[FixedEndian]): Unit = { + override def attrWrite(attr: AttrLikeSpec, id: Identifier, defEndian: Option[Endianness]): Unit = { attrParseIfHeader(id, attr.cond.ifExpr) - val io = normalIO + // Manage IO & seeking for ParseInstances + val io = attr match { + case pis: ParseInstanceSpec => + val io = pis.io match { + case None => normalIO + case Some(ex) => useIO(ex) + } + pis.pos.foreach { pos => + pushPos(io) + seek(io, pos) + } + io + case _ => + // no seeking required for sequence attributes + normalIO + } + + defEndian match { + case Some(_: CalcEndian) | Some(InheritedEndian) => + // FIXME: rename to indicate that it can be used for both parsing/writing + attrParseHybrid( + () => attrWrite0(id, attr, io, Some(LittleEndian)), + () => attrWrite0(id, attr, io, Some(BigEndian)) + ) + case None => + attrWrite0(id, attr, io, None) + case Some(fe: FixedEndian) => + attrWrite0(id, attr, io, Some(fe)) + } + attr match { + case pis: ParseInstanceSpec => + // Restore position, if applicable + if (pis.pos.isDefined) + popPos(io) + case _ => // no seeking required for sequence attributes + } + + attrParseIfFooter(attr.cond.ifExpr) + } + + def attrWrite0(id: Identifier, attr: AttrLikeSpec, io: String, defEndian: Option[FixedEndian]): Unit = { attr.cond.repeat match { case RepeatEos => condRepeatCommonHeader(id, io, attr.dataType) @@ -30,9 +71,8 @@ trait EveryWriteIsExpression extends LanguageCompiler with ObjectOrientedLanguag case NoRepeat => attrWrite2(id, attr.dataType, io, attr.cond.repeat, false, defEndian) } - - attrParseIfFooter(attr.cond.ifExpr) } + def attrWrite2( id: Identifier, dataType: DataType, diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala index 16189e8ed..50d697008 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala @@ -98,11 +98,15 @@ abstract class LanguageCompiler( def writeHeader(endian: Option[FixedEndian]): Unit = ??? def writeFooter(): Unit = ??? - def attrWrite(attr: AttrLikeSpec, id: Identifier, defEndian: Option[FixedEndian]): Unit = ??? + def writeInstanceHeader(instName: InstanceIdentifier): Unit = ??? + def writeInstanceFooter(): Unit = ??? + def attrWrite(attr: AttrLikeSpec, id: Identifier, defEndian: Option[Endianness]): Unit = ??? def runWriteCalc(): Unit = ??? def checkHeader(): Unit = ??? def checkFooter(): Unit = ??? + def checkInstanceHeader(instName: InstanceIdentifier): Unit = ??? + def checkInstanceFooter(): Unit = ??? def attrCheck(attr: AttrLikeSpec, id: Identifier): Unit = ??? def condIfSetNull(instName: Identifier): Unit = {} diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/UniversalFooter.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/UniversalFooter.scala index c1e3895c6..2beb55b67 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/UniversalFooter.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/UniversalFooter.scala @@ -17,7 +17,9 @@ trait UniversalFooter extends LanguageCompiler { def classConstructorFooter: Unit = universalFooter override def readFooter: Unit = universalFooter override def writeFooter: Unit = universalFooter + override def writeInstanceFooter: Unit = universalFooter override def checkFooter: Unit = universalFooter + override def checkInstanceFooter: Unit = universalFooter def condRepeatExprFooter = universalFooter def condRepeatEosFooter: Unit = universalFooter override def condRepeatCommonFooter: Unit = universalFooter From 070391c9451d8c1a211645da02f7d53c8b24ce24 Mon Sep 17 00:00:00 2001 From: Petr Pucil Date: Fri, 28 Oct 2022 13:55:53 +0200 Subject: [PATCH 51/90] Set write flags of parse instances at the beginning of `_write` --- .../io/kaitai/struct/ClassCompiler.scala | 35 +++++++++++++++---- .../struct/languages/JavaCompiler.scala | 17 +++++++++ .../components/LanguageCompiler.scala | 3 ++ 3 files changed, 49 insertions(+), 6 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/ClassCompiler.scala b/shared/src/main/scala/io/kaitai/struct/ClassCompiler.scala index e72e11271..cc86269a0 100644 --- a/shared/src/main/scala/io/kaitai/struct/ClassCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/ClassCompiler.scala @@ -79,7 +79,7 @@ class ClassCompiler( compileEagerRead(curClass.seq, curClass.meta.endian) if (config.readWrite) { - compileWrite(curClass.seq, curClass.meta.endian) + compileWrite(curClass.seq, curClass.instances, curClass.meta.endian) compileCheck(curClass.seq) } @@ -263,17 +263,17 @@ class ClassCompiler( } } - def compileWrite(seq: List[AttrSpec], endian: Option[Endianness]): Unit = { + def compileWrite(seq: List[AttrSpec], instances: Map[InstanceIdentifier, InstanceSpec], endian: Option[Endianness]): Unit = { endian match { case None | Some(_: FixedEndian) => - compileSeqWriteProc(seq, None) + compileSeqWriteProc(seq, instances, None) case Some(CalcEndian(_, _)) | Some(InheritedEndian) => lang.writeHeader(None) lang.runWriteCalc() lang.writeFooter() - compileSeqWriteProc(seq, Some(LittleEndian)) - compileSeqWriteProc(seq, Some(BigEndian)) + compileSeqWriteProc(seq, instances, Some(LittleEndian)) + compileSeqWriteProc(seq, instances, Some(BigEndian)) } } @@ -310,8 +310,9 @@ class ClassCompiler( lang.readFooter() } - def compileSeqWriteProc(seq: List[AttrSpec], defEndian: Option[FixedEndian]) = { + def compileSeqWriteProc(seq: List[AttrSpec], instances: Map[InstanceIdentifier, InstanceSpec], defEndian: Option[FixedEndian]) = { lang.writeHeader(defEndian) + compileSetInstanceWriteFlags(instances) compileSeqWrite(seq, defEndian) lang.writeFooter() } @@ -332,6 +333,16 @@ class ClassCompiler( } } + def compileSetInstanceWriteFlags(instances: Map[InstanceIdentifier, InstanceSpec]) = { + instances.foreach { case (instName, instSpec) => + instSpec match { + case _: ParseInstanceSpec => + lang.instanceSetWriteFlag(instName, true) + case _: ValueInstanceSpec => // do nothing + } + } + } + def compileSeqWrite(seq: List[AttrSpec], defEndian: Option[FixedEndian]) = { var wasUnaligned = false seq.foreach { (attr) => @@ -380,6 +391,12 @@ class ClassCompiler( lang.instanceHeader(className, instName, dataType, instSpec.isNullable) if (lang.innerDocstrings) compileInstanceDoc(instName, instSpec) + if (config.readWrite) + instSpec match { + case _: ParseInstanceSpec => + lang.instanceCheckWriteFlagAndWrite(instName) + case _: ValueInstanceSpec => // do nothing + } lang.instanceCheckCacheAndReturn(instName, dataType) instSpec match { @@ -421,6 +438,12 @@ class ClassCompiler( instSpec.isNullable } lang.instanceDeclaration(instName, instSpec.dataTypeComposite, isNullable) + if (config.readWrite) + instSpec match { + case _: ParseInstanceSpec => + lang.instanceWriteFlagDeclaration(instName) + case _: ValueInstanceSpec => // do nothing + } } def compileEnum(curClass: ClassSpec, enumColl: EnumSpec): Unit = diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala index cec79bc15..8df819b8b 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala @@ -745,6 +745,14 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts(s"private ${kaitaiType2JavaTypeBoxed(attrType)} ${idToStr(attrName)};") } + override def instanceWriteFlagDeclaration(attrName: InstanceIdentifier): Unit = { + out.puts(s"private boolean _write${idToSetterStr(attrName)} = false;") + } + + override def instanceSetWriteFlag(instName: InstanceIdentifier, value: Boolean): Unit = { + out.puts(s"_write${idToSetterStr(instName)} = ${expression(Ast.expr.Bool(value))};") + } + override def instanceHeader(className: String, instName: InstanceIdentifier, dataType: DataType, isNullable: Boolean): Unit = { out.puts(s"public ${kaitaiType2JavaTypeBoxed(dataType)} ${idToStr(instName)}() {") out.inc @@ -757,6 +765,15 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.dec } + override def instanceCheckWriteFlagAndWrite(instName: InstanceIdentifier): Unit = { + out.puts(s"if (_write${idToSetterStr(instName)}) {") + out.inc + out.puts(s"_write${idToSetterStr(instName)}();") + instanceSetWriteFlag(instName, false) + out.dec + out.puts("}") + } + override def instanceReturn(instName: InstanceIdentifier, attrType: DataType): Unit = { out.puts(s"return ${privateMemberName(instName)};") } diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala index 50d697008..ae7b2585f 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala @@ -141,12 +141,15 @@ abstract class LanguageCompiler( def instanceClear(instName: InstanceIdentifier): Unit = {} def instanceSetCalculated(instName: InstanceIdentifier): Unit = {} def instanceDeclaration(attrName: InstanceIdentifier, attrType: DataType, isNullable: Boolean): Unit = attributeDeclaration(attrName, attrType, isNullable) + def instanceWriteFlagDeclaration(attrName: InstanceIdentifier): Unit = ??? + def instanceSetWriteFlag(instName: InstanceIdentifier, value: Boolean): Unit = ??? def instanceHeader(className: List[String], instName: InstanceIdentifier, dataType: DataType, isNullable: Boolean): Unit def instanceFooter: Unit def instanceCheckCacheAndReturn(instName: InstanceIdentifier, dataType: DataType): Unit def instanceReturn(instName: InstanceIdentifier, attrType: DataType): Unit def instanceCalculate(instName: Identifier, dataType: DataType, value: Ast.expr) def instanceInvalidate(instName: InstanceIdentifier): Unit = ??? + def instanceCheckWriteFlagAndWrite(instName: InstanceIdentifier): Unit = ??? def enumDeclaration(curClass: List[String], enumName: String, enumColl: Seq[(Long, EnumValueSpec)]): Unit From 98978181af5e034690e8d881acf97f270ba9ad0e Mon Sep 17 00:00:00 2001 From: Petr Pucil Date: Fri, 28 Oct 2022 17:57:49 +0200 Subject: [PATCH 52/90] _write(): use special loops for each repetition type again Using a common loop (see https://github.com/kaitai-io/kaitai_struct_compiler/pull/182) would be fine if evaluating the user expressions in `repeat-expr` and `repeat-until` couldn't cause any side effects. However, it in fact *can* cause them by referring to positional instances, which need to be written at the moment they're first invoked so that the context of all `_io` properties matches the context in which they would be parsed. Hence the first invocation of instances has a side effect, and not invoking appropriate instances by not evaluating an expression (used in parsing) in writing would therefore introduce a discrepancy between parsing and serialization behavior. In general, if a user expression we know nothing about is used somewhere in parsing, we must use it in the same place during serialization. --- .../struct/languages/JavaCompiler.scala | 6 ++++- .../components/EveryWriteIsExpression.scala | 26 +++++++++++++------ 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala index 8df819b8b..660e1fd09 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala @@ -499,7 +499,7 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) importList.add("java.util.ArrayList") } - // used for all repetitions in _write() and _check() + // used for all repetitions in _check() override def condRepeatCommonHeader(id: Identifier, io: String, dataType: DataType): Unit = { out.puts(s"for (int i = 0; i < ${privateMemberName(id)}.size(); i++) {") out.inc @@ -529,6 +529,10 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts(s"${privateMemberName(id)}.add($tempVar);") } + override def handleAssignmentRepeatUntilIterator(expr: String): Unit = { + out.puts(s"${translator.doName(Identifier.ITERATOR)} = $expr;") + } + override def condRepeatUntilFooter(id: Identifier, io: String, dataType: DataType, untilExpr: expr): Unit = { typeProvider._currentIteratorType = Some(dataType) out.puts("i++;") diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala index e88830ff9..e2da014ee 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala @@ -57,19 +57,27 @@ trait EveryWriteIsExpression extends LanguageCompiler with ObjectOrientedLanguag def attrWrite0(id: Identifier, attr: AttrLikeSpec, io: String, defEndian: Option[FixedEndian]): Unit = { attr.cond.repeat match { case RepeatEos => + // we could use condRepeatEosHeader instead (we only deal with fixed-size streams when + // writing), but don't have to (there is no difference, because the `repeat: eos` repetition + // doesn't involve user expressions, unlike `repeat: expr` and `repeat: until`) condRepeatCommonHeader(id, io, attr.dataType) - attrWrite2(id, attr.dataType, io, attr.cond.repeat, false, defEndian) - condRepeatCommonFooter case RepeatExpr(repeatExpr: Ast.expr) => - condRepeatCommonHeader(id, io, attr.dataType) - attrWrite2(id, attr.dataType, io, attr.cond.repeat, false, defEndian) - condRepeatCommonFooter + condRepeatExprHeader(id, io, attr.dataType, repeatExpr) case RepeatUntil(untilExpr: Ast.expr) => - condRepeatCommonHeader(id, io, attr.dataType) - attrWrite2(id, attr.dataType, io, attr.cond.repeat, false, defEndian) + condRepeatUntilHeader(id, io, attr.dataType, untilExpr) + val expr = writeExprAsExpr(id, attr.cond.repeat, false) + handleAssignmentRepeatUntilIterator(translator.translate(expr)) + case NoRepeat => + } + attrWrite2(id, attr.dataType, io, attr.cond.repeat, false, defEndian) + attr.cond.repeat match { + case RepeatEos => condRepeatCommonFooter + case _: RepeatExpr => + condRepeatExprFooter + case RepeatUntil(untilExpr: Ast.expr) => + condRepeatUntilFooter(id, io, attr.dataType, untilExpr) case NoRepeat => - attrWrite2(id, attr.dataType, io, attr.cond.repeat, false, defEndian) } } @@ -284,6 +292,8 @@ trait EveryWriteIsExpression extends LanguageCompiler with ObjectOrientedLanguag def internalEnumIntType(basedOn: IntType): DataType + def handleAssignmentRepeatUntilIterator(expr: String): Unit + def attrPrimitiveWrite(io: String, expr: Ast.expr, dt: DataType, defEndian: Option[FixedEndian], exprTypeOpt: Option[DataType]): Unit def attrBytesLimitWrite(io: String, expr: Ast.expr, size: String, term: Int, padRight: Int): Unit def attrUserTypeInstreamWrite(io: String, expr: Ast.expr, t: DataType, exprType: DataType): Unit From 69e4a12dd77bbbb3c4955733be26d0bfac0f7e80 Mon Sep 17 00:00:00 2001 From: Petr Pucil Date: Fri, 28 Oct 2022 21:49:44 +0200 Subject: [PATCH 53/90] Java: make {_read,_write}{BE,LE}() `private` even if autoRead is off The possibility of exposing these internal methods as public was most likely an oversight. --- .../struct/languages/JavaCompiler.scala | 25 ++++++++----------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala index 660e1fd09..0c8cab3ac 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala @@ -211,26 +211,23 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } override def readHeader(endian: Option[FixedEndian], isEmpty: Boolean) = { - val readAccessAndType = if (!config.autoRead || config.readWrite) { - "public" - } else { - "private" - } - val suffix = endian match { - case Some(e) => Utils.upperUnderscoreCase(e.toSuffix) - case None => "" + endian match { + case Some(e) => + out.puts(s"private void _read${Utils.upperUnderscoreCase(e.toSuffix)}() {") + case None => + out.puts(s"${if (!config.autoRead) "public" else "private"} void _read() {") } - out.puts(s"$readAccessAndType void _read$suffix() {") out.inc } override def writeHeader(endian: Option[FixedEndian]): Unit = { - val suffix = endian match { - case Some(e) => Utils.upperUnderscoreCase(e.toSuffix) - case None => "" - } out.puts - out.puts(s"public void _write$suffix() {") + endian match { + case Some(e) => + out.puts(s"private void _write${Utils.upperUnderscoreCase(e.toSuffix)}() {") + case None => + out.puts("public void _write() {") + } out.inc } From 27101914a8a5cd1f4061e4b18be410ef367e15dd Mon Sep 17 00:00:00 2001 From: Petr Pucil Date: Fri, 28 Oct 2022 22:19:32 +0200 Subject: [PATCH 54/90] Fix instance write flags (previous attempt led to infinite recursion) --- .../main/scala/io/kaitai/struct/languages/JavaCompiler.scala | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala index 0c8cab3ac..0e7bc843a 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala @@ -241,6 +241,7 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts out.puts(s"public void _write${idToSetterStr(instName)}() {") out.inc + instanceSetWriteFlag(instName, false) } override def checkInstanceHeader(instName: InstanceIdentifier): Unit = { @@ -767,12 +768,10 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } override def instanceCheckWriteFlagAndWrite(instName: InstanceIdentifier): Unit = { - out.puts(s"if (_write${idToSetterStr(instName)}) {") + out.puts(s"if (_write${idToSetterStr(instName)})") out.inc out.puts(s"_write${idToSetterStr(instName)}();") - instanceSetWriteFlag(instName, false) out.dec - out.puts("}") } override def instanceReturn(instName: InstanceIdentifier, attrType: DataType): Unit = { From 7dce0c36149ea24a1ea8009c6d1b09b73e6516e9 Mon Sep 17 00:00:00 2001 From: Petr Pucil Date: Fri, 28 Oct 2022 22:21:13 +0200 Subject: [PATCH 55/90] Rename _write{,LE,BE}() methods to _write_Seq{,LE,BE}() --- .../io/kaitai/struct/languages/JavaCompiler.scala | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala index 0e7bc843a..943fd3590 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala @@ -201,11 +201,11 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.dec out.puts("} else if (_is_le) {") out.inc - out.puts("_writeLE();") + out.puts("_write_SeqLE();") out.dec out.puts("} else {") out.inc - out.puts("_writeBE();") + out.puts("_write_SeqBE();") out.dec out.puts("}") } @@ -224,9 +224,9 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts endian match { case Some(e) => - out.puts(s"private void _write${Utils.upperUnderscoreCase(e.toSuffix)}() {") + out.puts(s"private void _write_Seq${Utils.upperUnderscoreCase(e.toSuffix)}() {") case None => - out.puts("public void _write() {") + out.puts("public void _write_Seq() {") } out.inc } @@ -878,7 +878,7 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def attrUserTypeInstreamWrite(io: String, valueExpr: Ast.expr, dataType: DataType, exprType: DataType) = { val exprRaw = expression(valueExpr) val expr = castIfNeeded(exprRaw, exprType, dataType) - out.puts(s"$expr._write($io);") + out.puts(s"$expr._write_Seq($io);") } override def attrWriteStreamToStream(srcIo: String, dstIo: String) = From 826b24239b1b80c18bd565045bd42ac5a7f962ae Mon Sep 17 00:00:00 2001 From: Petr Pucil Date: Sun, 30 Oct 2022 00:29:49 +0200 Subject: [PATCH 56/90] Rename {writeExprAsExpr=>itemExpr}(), move to EveryReadIsExpression --- .../components/EveryReadIsExpression.scala | 13 ++++++++++ .../components/EveryWriteIsExpression.scala | 25 +++++-------------- .../languages/components/GenericChecks.scala | 4 +-- 3 files changed, 21 insertions(+), 21 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/EveryReadIsExpression.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/EveryReadIsExpression.scala index 7f803a816..1d1b231f7 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/EveryReadIsExpression.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/EveryReadIsExpression.scala @@ -200,4 +200,17 @@ trait EveryReadIsExpression attrDebugStart(instName, dataType, None, NoRepeat) handleAssignmentSimple(instName, expression(value)) } + + def itemExpr(id: Identifier, rep: RepeatSpec, isRaw: Boolean): Ast.expr = { + val astId = Ast.expr.InternalName(id) + rep match { + case NoRepeat => + astId + case _ => + Ast.expr.Subscript( + astId, + Ast.expr.Name(Ast.identifier(Identifier.INDEX)) + ) + } + } } diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala index e2da014ee..bf551501a 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala @@ -65,7 +65,7 @@ trait EveryWriteIsExpression extends LanguageCompiler with ObjectOrientedLanguag condRepeatExprHeader(id, io, attr.dataType, repeatExpr) case RepeatUntil(untilExpr: Ast.expr) => condRepeatUntilHeader(id, io, attr.dataType, untilExpr) - val expr = writeExprAsExpr(id, attr.cond.repeat, false) + val expr = itemExpr(id, attr.cond.repeat, false) handleAssignmentRepeatUntilIterator(translator.translate(expr)) case NoRepeat => } @@ -106,28 +106,15 @@ trait EveryWriteIsExpression extends LanguageCompiler with ObjectOrientedLanguag case t: StrFromBytesType => attrStrTypeWrite(id, t, io, rep, isRaw, exprTypeOpt) case t: EnumType => - val expr = writeExprAsExpr(id, rep, isRaw) + val expr = itemExpr(id, rep, isRaw) val exprType = internalEnumIntType(t.basedOn) attrPrimitiveWrite(io, Ast.expr.Attribute(expr, Ast.identifier("to_i")), t.basedOn, defEndian, Some(exprType)) case _ => - val expr = writeExprAsExpr(id, rep, isRaw) + val expr = itemExpr(id, rep, isRaw) attrPrimitiveWrite(io, expr, dataType, defEndian, exprTypeOpt) } } - def writeExprAsExpr(id: Identifier, rep: RepeatSpec, isRaw: Boolean): Ast.expr = { - val astId = Ast.expr.InternalName(id) - rep match { - case NoRepeat => - astId - case _ => - Ast.expr.Subscript( - astId, - Ast.expr.Name(Ast.identifier(Identifier.INDEX)) - ) - } - } - def attrBytesTypeWrite(id: Identifier, t: BytesType, io: String, rep: RepeatSpec, isRaw: Boolean, exprTypeOpt: Option[DataType]): Unit = { val idToWrite = t.process match { case Some(proc) => @@ -137,12 +124,12 @@ trait EveryWriteIsExpression extends LanguageCompiler with ObjectOrientedLanguag case None => id } - val expr = writeExprAsExpr(idToWrite, rep, isRaw) + val expr = itemExpr(idToWrite, rep, isRaw) attrBytesTypeWrite2(io, expr, t, exprTypeOpt) } def attrStrTypeWrite(id: Identifier, t: StrFromBytesType, io: String, rep: RepeatSpec, isRaw: Boolean, exprTypeOpt: Option[DataType]): Unit = { - val expr = exprStrToBytes(writeExprAsExpr(id, rep, isRaw), t.encoding) + val expr = exprStrToBytes(itemExpr(id, rep, isRaw), t.encoding) attrBytesTypeWrite2(io, expr, t.bytes, exprTypeOpt) } @@ -210,7 +197,7 @@ trait EveryWriteIsExpression extends LanguageCompiler with ObjectOrientedLanguag exprTypeOpt: Option[DataType] = None ) = { val exprType = exprTypeOpt.getOrElse(t) - val expr = writeExprAsExpr(id, rep, isRaw) + val expr = itemExpr(id, rep, isRaw) t match { case _: UserTypeInstream => diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/GenericChecks.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/GenericChecks.scala index 7a77b4adc..ef5b9e106 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/GenericChecks.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/GenericChecks.scala @@ -35,7 +35,7 @@ trait GenericChecks extends LanguageCompiler with EveryReadIsExpression with Eve } def attrCheck2(id: Identifier, dataType: DataType, io: String, repeat: RepeatSpec, isRaw: Boolean) = { - val item = writeExprAsExpr(id, repeat, isRaw) + val item = itemExpr(id, repeat, isRaw) dataType match { case t: BytesType => attrBytesCheck(item, t, idToMsg(id)) @@ -203,7 +203,7 @@ trait GenericChecks extends LanguageCompiler with EveryReadIsExpression with Eve handleAssignmentTempVar( dataType, translator.doName(Identifier.ITERATOR), - translator.translate(writeExprAsExpr(id, repeat, isRaw)) + translator.translate(itemExpr(id, repeat, isRaw)) ) typeProvider._currentIteratorType = Some(dataType) attrAssertEqual( From ca012b68c60796f08a587ef31795bfa984eb0487 Mon Sep 17 00:00:00 2001 From: Petr Pucil Date: Sun, 30 Oct 2022 00:33:46 +0200 Subject: [PATCH 57/90] Add _fetchInstances() method to recursively fetch parse instances --- .../io/kaitai/struct/ClassCompiler.scala | 16 ++++++ .../struct/languages/JavaCompiler.scala | 16 ++++++ .../languages/components/FetchInstances.scala | 53 +++++++++++++++++++ .../components/LanguageCompiler.scala | 6 +++ .../components/UniversalFooter.scala | 1 + 5 files changed, 92 insertions(+) create mode 100644 shared/src/main/scala/io/kaitai/struct/languages/components/FetchInstances.scala diff --git a/shared/src/main/scala/io/kaitai/struct/ClassCompiler.scala b/shared/src/main/scala/io/kaitai/struct/ClassCompiler.scala index cc86269a0..968ba293f 100644 --- a/shared/src/main/scala/io/kaitai/struct/ClassCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/ClassCompiler.scala @@ -78,6 +78,10 @@ class ClassCompiler( // Read method(s) compileEagerRead(curClass.seq, curClass.meta.endian) + compileFetchInstancesProc(curClass.seq ++ curClass.instances.values.collect { + case inst: AttrLikeSpec => inst + }) + if (config.readWrite) { compileWrite(curClass.seq, curClass.instances, curClass.meta.endian) compileCheck(curClass.seq) @@ -310,6 +314,12 @@ class ClassCompiler( lang.readFooter() } + def compileFetchInstancesProc(attrs: List[AttrLikeSpec]) = { + lang.fetchInstancesHeader() + compileFetchInstances(attrs) + lang.fetchInstancesFooter() + } + def compileSeqWriteProc(seq: List[AttrSpec], instances: Map[InstanceIdentifier, InstanceSpec], defEndian: Option[FixedEndian]) = { lang.writeHeader(defEndian) compileSetInstanceWriteFlags(instances) @@ -333,6 +343,12 @@ class ClassCompiler( } } + def compileFetchInstances(attrs: List[AttrLikeSpec]): Unit = { + attrs.foreach { (attr) => + lang.attrFetchInstances(attr, attr.id) + } + } + def compileSetInstanceWriteFlags(instances: Map[InstanceIdentifier, InstanceSpec]) = { instances.foreach { case (instName, instSpec) => instSpec match { diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala index 943fd3590..53a7ab332 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala @@ -15,6 +15,7 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) with UpperCamelCaseClasses with ObjectOrientedLanguage with EveryReadIsExpression + with FetchInstances with EveryWriteIsExpression with GenericChecks with UniversalFooter @@ -220,6 +221,21 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.inc } + override def fetchInstancesHeader(): Unit = { + out.puts + out.puts("public void _fetchInstances() {") + out.inc + } + + override def attrInvokeFetchInstances(baseExpr: Ast.expr, exprType: DataType, dataType: DataType): Unit = { + val expr = castIfNeeded(expression(baseExpr), exprType, dataType) + out.puts(s"$expr._fetchInstances();") + } + + override def attrInvokeInstance(instName: InstanceIdentifier): Unit = { + out.puts(s"${publicMemberName(instName)}();") + } + override def writeHeader(endian: Option[FixedEndian]): Unit = { out.puts endian match { diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/FetchInstances.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/FetchInstances.scala new file mode 100644 index 000000000..6bf98b092 --- /dev/null +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/FetchInstances.scala @@ -0,0 +1,53 @@ +package io.kaitai.struct.languages.components + +import io.kaitai.struct.exprlang.Ast +import io.kaitai.struct.format._ +import io.kaitai.struct.datatype.DataType +import io.kaitai.struct.datatype.DataType._ + +trait FetchInstances extends LanguageCompiler with ObjectOrientedLanguage with EveryReadIsExpression { + override def attrFetchInstances(attr: AttrLikeSpec, id: Identifier): Unit = { + attrParseIfHeader(id, attr.cond.ifExpr) + + val io = normalIO + + id match { + case instName: InstanceIdentifier => + attrInvokeInstance(instName) + case _ => + } + + if (attr.cond.repeat != NoRepeat) + condRepeatCommonHeader(id, io, attr.dataType) + + attrFetchInstances2(id, attr.dataType, attr.cond.repeat) + + if (attr.cond.repeat != NoRepeat) + condRepeatCommonFooter + + attrParseIfFooter(attr.cond.ifExpr) + } + + def attrFetchInstances2(id: Identifier, dataType: DataType, rep: RepeatSpec, exprTypeOpt: Option[DataType] = None): Unit = { + dataType match { + case _: UserType => + val exprType = exprTypeOpt.getOrElse(dataType) + attrInvokeFetchInstances(itemExpr(id, rep, false), exprType, dataType) + case st: SwitchType => + attrSwitchTypeFetchInstances(id, st.on, st.cases, rep, st.combinedType) + case _ => + } + } + + def attrSwitchTypeFetchInstances(id: Identifier, on: Ast.expr, cases: Map[Ast.expr, DataType], rep: RepeatSpec, assignType: DataType): Unit = { + switchCases[DataType](id, on, cases, + (dataType) => { + attrFetchInstances2(id, dataType, rep, Some(assignType)) + }, + (dataType) => { + // TODO: process switchBytesOnlyAsRaw + attrFetchInstances2(id, dataType, rep, Some(assignType)) + } + ) + } +} diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala index ae7b2585f..9b827883c 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala @@ -96,6 +96,12 @@ abstract class LanguageCompiler( def attrInit(attr: AttrLikeSpec): Unit = {} def attrDestructor(attr: AttrLikeSpec, id: Identifier): Unit = {} + def attrFetchInstances(attr: AttrLikeSpec, id: Identifier): Unit = {} + def fetchInstancesHeader(): Unit = {} + def fetchInstancesFooter(): Unit = {} + def attrInvokeFetchInstances(baseExpr: Ast.expr, exprType: DataType, dataType: DataType): Unit = ??? + def attrInvokeInstance(instName: InstanceIdentifier): Unit = ??? + def writeHeader(endian: Option[FixedEndian]): Unit = ??? def writeFooter(): Unit = ??? def writeInstanceHeader(instName: InstanceIdentifier): Unit = ??? diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/UniversalFooter.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/UniversalFooter.scala index 2beb55b67..2f1de52de 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/UniversalFooter.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/UniversalFooter.scala @@ -16,6 +16,7 @@ trait UniversalFooter extends LanguageCompiler { def classFooter(name: String): Unit = universalFooter def classConstructorFooter: Unit = universalFooter override def readFooter: Unit = universalFooter + override def fetchInstancesFooter: Unit = universalFooter override def writeFooter: Unit = universalFooter override def writeInstanceFooter: Unit = universalFooter override def checkFooter: Unit = universalFooter From 3b08efe3315874c342b8d096482a8863faf991a4 Mon Sep 17 00:00:00 2001 From: Petr Pucil Date: Sat, 26 Nov 2022 15:27:48 +0100 Subject: [PATCH 58/90] Write substreams to parent only after all fields have been written This is needed for general support of `instances` if we want to write all substreams only once. See https://github.com/kaitai-io/kaitai_struct_java_runtime/commit/c0a246b2f22ea61ccc64d6786b670330ed827200 for the corresponding runtime library changes. Only `size` and `size-eos` streams without `process` are supported so far. It does not work for substreams using `process` or only delimited by `terminator`, because in such cases we don't know how long the substream should be created. So the user will have to provide the substream length via a special setter - this will be addressed later. --- .../struct/languages/JavaCompiler.scala | 27 +++++++++++++++ .../components/EveryWriteIsExpression.scala | 33 ++++++++++++------- .../components/LanguageCompiler.scala | 8 +++++ 3 files changed, 56 insertions(+), 12 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala index 53a7ab332..96d844ec7 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala @@ -400,9 +400,33 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) ioName } + override def exprIORemainingSize(io: String): String = + s"$io.size() - $io.pos()" + override def allocateIOGrowing(varName: Identifier): String = allocateIOFixed(varName, "100000") // FIXME to use real growing buffer + override def subIOWriteBackHeader(subIO: String, io: String): String = { + val parentIoName = "parent" + out.puts(s"$subIO.setWriteBackHandler(new $kstreamName.WriteBackHandler($io.pos()) {") + out.inc + out.puts("@Override") + out.puts(s"protected void write($kstreamName $parentIoName) {") + out.inc + + parentIoName + } + + override def subIOWriteBackFooter: Unit = { + out.dec + out.puts("}") + out.dec + out.puts("});") + } + + override def addChildIO(io: String, childIO: String): Unit = + out.puts(s"$io.addChildStream($childIO);") + def getRawIdExpr(varName: Identifier, rep: RepeatSpec): String = { val memberName = idToStr(varName) rep match { @@ -422,6 +446,9 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def seek(io: String, pos: Ast.expr): Unit = out.puts(s"$io.seek(${expression(pos)});") + override def seekRelative(io: String, relPos: String): Unit = + out.puts(s"$io.seek($io.pos() + ($relPos));") + override def popPos(io: String): Unit = out.puts(s"$io.seek(_pos);") diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala index bf551501a..f701c1206 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala @@ -207,20 +207,29 @@ trait EveryWriteIsExpression extends LanguageCompiler with ObjectOrientedLanguag val byteType = knownSizeType.bytes byteType.process match { case None => - byteType match { + val size = byteType match { case blt: BytesLimitType => - this match { - // case thisStore: AllocateAndStoreIO => - // val ourIO = thisStore.allocateIO(rawId, rep) - // Utils.addUniqueAttr(extraAttrs, AttrSpec(List(), ourIO, KaitaiStreamType)) - // privateMemberName(ourIO) - case thisLocal: AllocateIOLocalVar => - val ioFixed = thisLocal.allocateIOFixed(rawId, translator.translate(blt.size)) - attrUserTypeInstreamWrite(ioFixed, expr, t, exprType) - attrWriteStreamToStream(ioFixed, io) + translator.translate(blt.size) + case _: BytesEosType => + exprIORemainingSize(io) + } + this match { + // case thisStore: AllocateAndStoreIO => + // val ourIO = thisStore.allocateIO(rawId, rep) + // Utils.addUniqueAttr(extraAttrs, AttrSpec(List(), ourIO, KaitaiStreamType)) + // privateMemberName(ourIO) + case thisLocal: AllocateIOLocalVar => + val ioFixed = thisLocal.allocateIOFixed(rawId, size) + + { + val parentIO = subIOWriteBackHeader(ioFixed, io) + attrWriteStreamToStream(ioFixed, parentIO) + subIOWriteBackFooter } - case _ => - attrUserTypeInstreamWrite(io, expr, t, exprType) + + addChildIO(io, ioFixed) + seekRelative(io, size) + attrUserTypeInstreamWrite(ioFixed, expr, t, exprType) } case Some(process) => byteType match { diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala index 9b827883c..d60ba234b 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala @@ -140,9 +140,17 @@ abstract class LanguageCompiler( def useIO(ioEx: Ast.expr): String def pushPos(io: String): Unit def seek(io: String, pos: Ast.expr): Unit + def seekRelative(io: String, relPos: String): Unit = ??? def popPos(io: String): Unit def alignToByte(io: String): Unit + def exprIORemainingSize(io: String): String = ??? + + def subIOWriteBackHeader(subIO: String, io: String): String = ??? + def subIOWriteBackFooter: Unit = ??? + + def addChildIO(io: String, childIO: String): Unit = ??? + def instanceDeclHeader(className: List[String]): Unit = {} def instanceClear(instName: InstanceIdentifier): Unit = {} def instanceSetCalculated(instName: InstanceIdentifier): Unit = {} From 544cad06a25d95b9b7b8630d303bbf2e9fd52bee Mon Sep 17 00:00:00 2001 From: Petr Pucil Date: Sat, 26 Nov 2022 16:26:24 +0100 Subject: [PATCH 59/90] Add set{Inst}_ToWrite() setter to select what instances to write --- .../scala/io/kaitai/struct/ClassCompiler.scala | 3 ++- .../io/kaitai/struct/languages/JavaCompiler.scala | 15 ++++++++++++--- .../languages/components/LanguageCompiler.scala | 4 +++- 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/ClassCompiler.scala b/shared/src/main/scala/io/kaitai/struct/ClassCompiler.scala index 968ba293f..e389454fb 100644 --- a/shared/src/main/scala/io/kaitai/struct/ClassCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/ClassCompiler.scala @@ -353,7 +353,7 @@ class ClassCompiler( instances.foreach { case (instName, instSpec) => instSpec match { case _: ParseInstanceSpec => - lang.instanceSetWriteFlag(instName, true) + lang.instanceSetWriteFlag(instName) case _: ValueInstanceSpec => // do nothing } } @@ -432,6 +432,7 @@ class ClassCompiler( instSpec match { case pi: ParseInstanceSpec => lang.attributeSetter(instName, dataType, instSpec.isNullable) + lang.instanceToWriteSetter(instName) lang.writeInstanceHeader(instName) lang.attrWrite(pi, instName, endian) lang.writeInstanceFooter diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala index 96d844ec7..d2efa72b4 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala @@ -257,7 +257,7 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts out.puts(s"public void _write${idToSetterStr(instName)}() {") out.inc - instanceSetWriteFlag(instName, false) + instanceClearWriteFlag(instName) } override def checkInstanceHeader(instName: InstanceIdentifier): Unit = { @@ -792,10 +792,19 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def instanceWriteFlagDeclaration(attrName: InstanceIdentifier): Unit = { out.puts(s"private boolean _write${idToSetterStr(attrName)} = false;") + out.puts(s"private boolean _toWrite${idToSetterStr(attrName)} = true;") } - override def instanceSetWriteFlag(instName: InstanceIdentifier, value: Boolean): Unit = { - out.puts(s"_write${idToSetterStr(instName)} = ${expression(Ast.expr.Bool(value))};") + override def instanceSetWriteFlag(instName: InstanceIdentifier): Unit = { + out.puts(s"_write${idToSetterStr(instName)} = _toWrite${idToSetterStr(instName)};") + } + + override def instanceClearWriteFlag(instName: InstanceIdentifier): Unit = { + out.puts(s"_write${idToSetterStr(instName)} = false;") + } + + override def instanceToWriteSetter(instName: InstanceIdentifier): Unit = { + out.puts(s"public void set${idToSetterStr(instName)}_ToWrite(boolean _v) { _toWrite${idToSetterStr(instName)} = _v; }") } override def instanceHeader(className: String, instName: InstanceIdentifier, dataType: DataType, isNullable: Boolean): Unit = { diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala index d60ba234b..623621cc8 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala @@ -156,7 +156,9 @@ abstract class LanguageCompiler( def instanceSetCalculated(instName: InstanceIdentifier): Unit = {} def instanceDeclaration(attrName: InstanceIdentifier, attrType: DataType, isNullable: Boolean): Unit = attributeDeclaration(attrName, attrType, isNullable) def instanceWriteFlagDeclaration(attrName: InstanceIdentifier): Unit = ??? - def instanceSetWriteFlag(instName: InstanceIdentifier, value: Boolean): Unit = ??? + def instanceSetWriteFlag(instName: InstanceIdentifier): Unit = ??? + def instanceClearWriteFlag(instName: InstanceIdentifier): Unit = ??? + def instanceToWriteSetter(instName: InstanceIdentifier): Unit = ??? def instanceHeader(className: List[String], instName: InstanceIdentifier, dataType: DataType, isNullable: Boolean): Unit def instanceFooter: Unit def instanceCheckCacheAndReturn(instName: InstanceIdentifier, dataType: DataType): Unit From 11da92f538df9260f21313044ad104039d19f5a8 Mon Sep 17 00:00:00 2001 From: Petr Pucil Date: Fri, 23 Dec 2022 19:59:23 +0100 Subject: [PATCH 60/90] Add support for writing substreams with `process` and/or `terminator` --- .../io/kaitai/struct/ClassTypeProvider.scala | 17 ++- .../io/kaitai/struct/format/Identifier.scala | 8 ++ .../struct/languages/JavaCompiler.scala | 82 ++++++++++-- .../components/EveryReadIsExpression.scala | 30 +++++ .../components/EveryWriteIsExpression.scala | 120 +++++++++++++----- .../languages/components/ExtraAttrs.scala | 49 ++++++- .../components/LanguageCompiler.scala | 4 +- 7 files changed, 256 insertions(+), 54 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/ClassTypeProvider.scala b/shared/src/main/scala/io/kaitai/struct/ClassTypeProvider.scala index 9bc0f7d6f..c376c985e 100644 --- a/shared/src/main/scala/io/kaitai/struct/ClassTypeProvider.scala +++ b/shared/src/main/scala/io/kaitai/struct/ClassTypeProvider.scala @@ -59,14 +59,17 @@ class ClassTypeProvider(classSpecs: ClassSpecs, var topClass: ClassSpec) extends case SpecialIdentifier(name) => return determineType(inClass, name) case RawIdentifier(innerId) => { val innerType = determineType(innerId) - val (isArray, singleType: DataType) = innerType match { + val (isArray, itemType: DataType) = innerType match { case at: ArrayType => (true, at.elType) - case st: SwitchType => (false, st.cases.collectFirst { + case t => (false, t) + } + val singleType: DataType = itemType match { + case st: SwitchType => st.cases.collectFirst { case (_, caseType) if caseType.isInstanceOf[BytesType] || caseType.isInstanceOf[UserTypeFromBytes] => caseType - }.get) - case _ => (false, innerType) + }.get + case t => t } /** see [[languages.components.ExtraAttrs$]] for possible types */ val bytesType = singleType match { @@ -75,6 +78,12 @@ class ClassTypeProvider(classSpecs: ClassSpecs, var topClass: ClassSpec) extends } return if (isArray) ArrayTypeInStream(bytesType) else bytesType } + case OuterSizeIdentifier(innerId) => + val singleType = CalcIntType + return if (determineType(innerId).isInstanceOf[ArrayType]) ArrayTypeInStream(singleType) else singleType + case InnerSizeIdentifier(innerId) => + val singleType = CalcIntType + return if (determineType(innerId).isInstanceOf[ArrayType]) ArrayTypeInStream(singleType) else singleType case _ => // do nothing } throw new FieldNotFoundError(attrId.humanReadable, inClass) diff --git a/shared/src/main/scala/io/kaitai/struct/format/Identifier.scala b/shared/src/main/scala/io/kaitai/struct/format/Identifier.scala index 91dd0118f..92b3e43a4 100644 --- a/shared/src/main/scala/io/kaitai/struct/format/Identifier.scala +++ b/shared/src/main/scala/io/kaitai/struct/format/Identifier.scala @@ -95,6 +95,14 @@ case class IoStorageIdentifier(innerId: Identifier) extends Identifier { override def humanReadable: String = s"io(${innerId.humanReadable})" } +case class OuterSizeIdentifier(innerId: Identifier) extends Identifier { + override def humanReadable: String = s"outerSize(${innerId.humanReadable})" +} + +case class InnerSizeIdentifier(innerId: Identifier) extends Identifier { + override def humanReadable: String = s"innerSize(${innerId.humanReadable})" +} + case class InstanceIdentifier(name: String) extends Identifier { Identifier.checkIdentifier(name) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala index d2efa72b4..8844829a1 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala @@ -40,6 +40,10 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } } + /** See [[subIOWriteBackHeader]] => the code generated when `true` will be inside the definition + * of the "writeBackHandler" callback function. */ + private var inSubIOWriteBackHandler = false; + override def universalFooter: Unit = { out.dec out.puts("}") @@ -343,26 +347,32 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) handleAssignment(varDest, expr, rep, false) } - // TODO: merge with attrProcess above (there is currently 99.9% duplication) override def attrUnprocess(proc: ProcessExpr, varSrc: Identifier, varDest: Identifier, rep: RepeatSpec, dataType: BytesType, exprTypeOpt: Option[DataType]): Unit = { val exprType = exprTypeOpt.getOrElse(dataType) - val srcExprRaw = getRawIdExpr(varSrc, rep) + val srcExprRaw = varSrc match { + // use `_raw_items[_raw_items.size - 1]` + case _: RawIdentifier => getRawIdExpr(varSrc, rep) + // but `items[_index]` + case _ => expression(itemExpr(varSrc, rep, false)) + } val srcExpr = castIfNeeded(srcExprRaw, exprType, dataType) val expr = proc match { case ProcessXor(xorValue) => + val argStr = if (inSubIOWriteBackHandler) "_processXorArg" else expression(xorValue) val xorValueStr = translator.detectType(xorValue) match { - case _: IntType => translator.doCast(xorValue, Int1Type(true)) - case _ => expression(xorValue) + case _: IntType => castIfNeeded(argStr, AnyType, Int1Type(true)) + case _ => argStr } s"$kstreamName.processXor($srcExpr, $xorValueStr)" case ProcessZlib => s"$kstreamName.unprocessZlib($srcExpr)" case ProcessRotate(isLeft, rotValue) => + val argStr = if (inSubIOWriteBackHandler) "_processRotateArg" else expression(rotValue) val expr = if (!isLeft) { - expression(rotValue) + argStr } else { - s"8 - (${expression(rotValue)})" + s"8 - ($argStr)" } s"$kstreamName.processRotateLeft($srcExpr, $expr, 1)" case ProcessCustom(name, args) => @@ -371,12 +381,33 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) (if (namespace.nonEmpty) "." else "") + type2class(name.last) val procName = s"_process_${idToStr(varSrc)}" - out.puts(s"$procClass $procName = new $procClass(${args.map(expression).mkString(", ")});") + if (!inSubIOWriteBackHandler) { + out.puts(s"$procClass $procName = new $procClass(${args.map(expression).mkString(", ")});") + } s"$procName.encode($srcExpr)" } handleAssignment(varDest, expr, rep, false) } + override def attrUnprocessPrepareBeforeSubIOHandler(proc: ProcessExpr, varSrc: Identifier): Unit = { + proc match { + case ProcessXor(xorValue) => + val dataType = translator.detectType(xorValue) + out.puts(s"final ${kaitaiType2JavaType(dataType)} _processXorArg = ${expression(xorValue)};") + case ProcessRotate(_, rotValue) => + val dataType = translator.detectType(rotValue) + out.puts(s"final ${kaitaiType2JavaType(dataType)} _processRotateArg = ${expression(rotValue)};") + case ProcessZlib => // no process arguments + case ProcessCustom(name, args) => + val namespace = name.init.mkString(".") + val procClass = namespace + + (if (namespace.nonEmpty) "." else "") + + type2class(name.last) + val procName = s"_process_${idToStr(varSrc)}" + out.puts(s"final $procClass $procName = new $procClass(${args.map(expression).mkString(", ")});") + } + } + override def allocateIO(varName: Identifier, rep: RepeatSpec): String = { val javaName = idToStr(varName) @@ -406,18 +437,23 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def allocateIOGrowing(varName: Identifier): String = allocateIOFixed(varName, "100000") // FIXME to use real growing buffer - override def subIOWriteBackHeader(subIO: String, io: String): String = { + override def subIOWriteBackHeader(subIO: String): String = { val parentIoName = "parent" - out.puts(s"$subIO.setWriteBackHandler(new $kstreamName.WriteBackHandler($io.pos()) {") + out.puts(s"final ${type2class(typeProvider.nowClass.name.last)} _this = this;") + out.puts(s"$subIO.setWriteBackHandler(new $kstreamName.WriteBackHandler(_pos2) {") out.inc out.puts("@Override") out.puts(s"protected void write($kstreamName $parentIoName) {") out.inc + inSubIOWriteBackHandler = true + parentIoName } override def subIOWriteBackFooter: Unit = { + inSubIOWriteBackHandler = false + out.dec out.puts("}") out.dec @@ -428,7 +464,7 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts(s"$io.addChildStream($childIO);") def getRawIdExpr(varName: Identifier, rep: RepeatSpec): String = { - val memberName = idToStr(varName) + val memberName = privateMemberName(varName) rep match { case NoRepeat => memberName case _ => s"$memberName.get($memberName.size() - 1)" @@ -443,6 +479,9 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def pushPos(io: String): Unit = out.puts(s"long _pos = $io.pos();") + override def pushPosForSubIOWriteBackHandler(io: String): Unit = + out.puts(s"long _pos2 = $io.pos();") + override def seek(io: String, pos: Ast.expr): Unit = out.puts(s"$io.seek(${expression(pos)});") @@ -508,9 +547,26 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts(s"${privateMemberName(RawIdentifier(id))} = new ArrayList();") if (needRaw.level >= 2) out.puts(s"${privateMemberName(RawIdentifier(RawIdentifier(id)))} = new ArrayList();") + if (config.readWrite) { + dataType match { + case utb: UserTypeFromBytes => + if (writeNeedsOuterSize(utb)) + out.puts(s"${privateMemberName(OuterSizeIdentifier(id))} = new ArrayList();") + if (writeNeedsInnerSize(utb)) + out.puts(s"${privateMemberName(InnerSizeIdentifier(id))} = new ArrayList();") + case _ => // do nothing + } + } out.puts(s"${privateMemberName(id)} = new ${kaitaiType2JavaType(ArrayTypeInStream(dataType))}();") } + override def condRepeatCommonWriteInit(id: Identifier, dataType: DataType, needRaw: NeedRaw): Unit = { + if (needRaw.level >= 1) + out.puts(s"${privateMemberName(RawIdentifier(id))} = new ArrayList();") + if (needRaw.level >= 2) + out.puts(s"${privateMemberName(RawIdentifier(RawIdentifier(id)))} = new ArrayList();") + } + override def condRepeatEosHeader(id: Identifier, io: String, dataType: DataType): Unit = { out.puts("{") out.inc @@ -963,10 +1019,12 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) case NumberedIdentifier(idx) => s"_${NumberedIdentifier.TEMPLATE}$idx" case InstanceIdentifier(name) => Utils.upperCamelCase(name) case RawIdentifier(innerId) => "_raw_" + idToSetterStr(innerId) + case OuterSizeIdentifier(innerId) => s"${idToSetterStr(innerId)}_OuterSize" + case InnerSizeIdentifier(innerId) => s"${idToSetterStr(innerId)}_InnerSize" } } - override def privateMemberName(id: Identifier): String = s"this.${idToStr(id)}" + override def privateMemberName(id: Identifier): String = s"${if (inSubIOWriteBackHandler) "_" else ""}this.${idToStr(id)}" override def localTemporaryName(id: Identifier): String = s"_t_${idToStr(id)}" @@ -1128,6 +1186,8 @@ object JavaCompiler extends LanguageCompilerStatic case NumberedIdentifier(idx) => s"_${NumberedIdentifier.TEMPLATE}$idx" case InstanceIdentifier(name) => Utils.lowerCamelCase(name) case RawIdentifier(innerId) => s"_raw_${idToStr(innerId)}" + case OuterSizeIdentifier(innerId) => s"${idToStr(innerId)}_OuterSize" + case InnerSizeIdentifier(innerId) => s"${idToStr(innerId)}_InnerSize" } def publicMemberName(id: Identifier) = idToStr(id) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/EveryReadIsExpression.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/EveryReadIsExpression.scala index 1d1b231f7..53d7c5039 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/EveryReadIsExpression.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/EveryReadIsExpression.scala @@ -108,6 +108,36 @@ trait EveryReadIsExpression attrParse2(rawId, byteType, io, rep, true, defEndian) + if (config.readWrite) { + if (writeNeedsOuterSize(knownSizeType)) { + /** @note Must be kept in sync with [[attrBytesTypeParse]] */ + val rawRawId = knownSizeType.process match { + case None => rawId + case Some(_) => RawIdentifier(rawId) + } + val item = itemExpr(rawRawId, rep, false) + val itemSizeExprStr = expression(Ast.expr.Attribute(item, Ast.identifier("size"))) + /** FIXME: cannot use [[handleAssignment]] because [[handleAssignmentRepeatUntil]] + * always tries to assign the value to the [[Identifier.ITERATOR]] variable */ + if (rep == NoRepeat) { + handleAssignmentSimple(OuterSizeIdentifier(id), itemSizeExprStr) + } else { + handleAssignmentRepeatEos(OuterSizeIdentifier(id), itemSizeExprStr) + } + } + if (writeNeedsInnerSize(knownSizeType)) { + val item = itemExpr(rawId, rep, false) + val itemSizeExprStr = expression(Ast.expr.Attribute(item, Ast.identifier("size"))) + /** FIXME: cannot use [[handleAssignment]] because [[handleAssignmentRepeatUntil]] + * always tries to assign the value to the [[Identifier.ITERATOR]] variable */ + if (rep == NoRepeat) { + handleAssignmentSimple(InnerSizeIdentifier(id), itemSizeExprStr) + } else { + handleAssignmentRepeatEos(InnerSizeIdentifier(id), itemSizeExprStr) + } + } + } + val extraType = rep match { case NoRepeat => byteType case _ => ArrayTypeInStream(byteType) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala index f701c1206..9319b9236 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala @@ -55,6 +55,8 @@ trait EveryWriteIsExpression extends LanguageCompiler with ObjectOrientedLanguag } def attrWrite0(id: Identifier, attr: AttrLikeSpec, io: String, defEndian: Option[FixedEndian]): Unit = { + if (attr.cond.repeat != NoRepeat) + condRepeatCommonWriteInit(id, attr.dataType, needRaw(attr.dataType)) attr.cond.repeat match { case RepeatEos => // we could use condRepeatEosHeader instead (we only deal with fixed-size streams when @@ -124,7 +126,36 @@ trait EveryWriteIsExpression extends LanguageCompiler with ObjectOrientedLanguag case None => id } - val expr = itemExpr(idToWrite, rep, isRaw) + val expr = if (idToWrite.isInstanceOf[RawIdentifier] && rep != NoRepeat) { + // NOTE: This special handling isn't normally needed and one can just use + // `itemExpr(idToWrite, rep, isRaw)` as usual. The `itemExpr` method assumes that the + // expression it's supposed to generate will be used in a loop where the iteration + // variable `Identifier.INDEX` is available (usually called just `i`) and uses it. This + // is a good default, but it doesn't work if the expression is used between + // `subIOWriteBackHeader` and `subIOWriteBackFooter` (see `attrUserTypeWrite` below), + // because in Java the loop control variable `i` is not "final" or "effectively final". + // + // The workaround is to change the expression so that it doesn't depend on the `i` + // variable. We can do that here, because the `RawIdentifier(...)` array starts empty + // before the loop and each element is added by `attrUnprocess` in each loop iteration - + // so the current item is just the last entry in the `RawIdentifier(...)` array. + // + // See test ProcessRepeatUsertype that requires this. + val astId = Ast.expr.InternalName(idToWrite) + Ast.expr.Subscript( + astId, + Ast.expr.BinOp( + Ast.expr.Attribute( + astId, + Ast.identifier("size") + ), + Ast.operator.Sub, + Ast.expr.IntNum(1) + ) + ) + } else { + itemExpr(idToWrite, rep, isRaw) + } attrBytesTypeWrite2(io, expr, t, exprTypeOpt) } @@ -202,46 +233,64 @@ trait EveryWriteIsExpression extends LanguageCompiler with ObjectOrientedLanguag t match { case _: UserTypeInstream => attrUserTypeInstreamWrite(io, expr, t, exprType) - case knownSizeType: UserTypeFromBytes => + case utb: UserTypeFromBytes => val rawId = RawIdentifier(id) - val byteType = knownSizeType.bytes - byteType.process match { - case None => - val size = byteType match { - case blt: BytesLimitType => - translator.translate(blt.size) - case _: BytesEosType => - exprIORemainingSize(io) - } - this match { - // case thisStore: AllocateAndStoreIO => - // val ourIO = thisStore.allocateIO(rawId, rep) - // Utils.addUniqueAttr(extraAttrs, AttrSpec(List(), ourIO, KaitaiStreamType)) - // privateMemberName(ourIO) - case thisLocal: AllocateIOLocalVar => - val ioFixed = thisLocal.allocateIOFixed(rawId, size) + val byteType = utb.bytes - { - val parentIO = subIOWriteBackHeader(ioFixed, io) - attrWriteStreamToStream(ioFixed, parentIO) - subIOWriteBackFooter - } + /** @note Must be kept in sync with [[ExtraAttrs.writeNeedsOuterSize]] */ + val outerSize = byteType match { + case blt: BytesLimitType => + translator.translate(blt.size) + case _: BytesEosType => + exprIORemainingSize(io) + case _: BytesTerminatedType => + translator.translate(itemExpr(OuterSizeIdentifier(id), rep, isRaw)) + } - addChildIO(io, ioFixed) - seekRelative(io, size) - attrUserTypeInstreamWrite(ioFixed, expr, t, exprType) - } - case Some(process) => + /** @note Must be kept in sync with [[ExtraAttrs.writeNeedsInnerSize]] */ + val innerSize = if (writeNeedsInnerSize(utb)) { + translator.translate(itemExpr(InnerSizeIdentifier(id), rep, isRaw)) + } else { + outerSize + } + + this match { + // case thisStore: AllocateAndStoreIO => + // val ourIO = thisStore.allocateIO(rawId, rep) + // Utils.addUniqueAttr(extraAttrs, AttrSpec(List(), ourIO, KaitaiStreamType)) + // privateMemberName(ourIO) + case thisLocal: AllocateIOLocalVar => + val ioFixed = thisLocal.allocateIOFixed(rawId, innerSize) + addChildIO(io, ioFixed) + + blockScopeHeader + + pushPosForSubIOWriteBackHandler(io) + seekRelative(io, outerSize) byteType match { - case blt: BytesLimitType => - this match { - case thisLocal: AllocateIOLocalVar => - val ioFixed = thisLocal.allocateIOFixed(rawId, translator.translate(blt.size)) - attrUserTypeInstreamWrite(ioFixed, expr, t, exprType) - handleAssignment(rawId, exprStreamToByteArray(ioFixed), rep, isRaw) - attrBytesTypeWrite(rawId, byteType, io, rep, isRaw, exprTypeOpt) + case t: BytesTerminatedType => + // FIXME: does not take `eos-error: false` into account (assumes `eos-error: true`) + if (!t.include && t.consume) { + // terminator can only be 1 byte long at the moment + seekRelative(io, expression(Ast.expr.IntNum(1))) } + case _ => // do nothing } + + byteType.process.foreach { (process) => + attrUnprocessPrepareBeforeSubIOHandler(process, rawId) + } + + { + val parentIO = subIOWriteBackHeader(ioFixed) + handleAssignment(rawId, exprStreamToByteArray(ioFixed), rep, true) + attrBytesTypeWrite(rawId, byteType, parentIO, rep, isRaw, exprTypeOpt) + subIOWriteBackFooter + } + + blockScopeFooter + + attrUserTypeInstreamWrite(ioFixed, expr, t, exprType) } } } @@ -297,4 +346,5 @@ trait EveryWriteIsExpression extends LanguageCompiler with ObjectOrientedLanguag def exprStreamToByteArray(ioFixed: String): String def attrUnprocess(proc: ProcessExpr, varSrc: Identifier, varDest: Identifier, rep: RepeatSpec, dt: BytesType, exprTypeOpt: Option[DataType]): Unit + def attrUnprocessPrepareBeforeSubIOHandler(proc: ProcessExpr, varSrc: Identifier): Unit } diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/ExtraAttrs.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/ExtraAttrs.scala index 90f19d3a9..2e6c17507 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/ExtraAttrs.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/ExtraAttrs.scala @@ -3,22 +3,49 @@ package io.kaitai.struct.languages.components import io.kaitai.struct.datatype.DataType import io.kaitai.struct.datatype.DataType._ import io.kaitai.struct.format._ +import io.kaitai.struct.RuntimeConfig /** * Trait to be implemented by all [[LanguageCompiler]] compilers: supplies extra attributes * when we'll be allocating new IOs. */ trait ExtraAttrs { + val config: RuntimeConfig + def extraAttrForIO(id: Identifier, rep: RepeatSpec): List[AttrSpec] + def writeNeedsOuterSize(utb: UserTypeFromBytes): Boolean = { + utb.bytes match { + case _: BytesTerminatedType => true + case _ => false + } + } + def writeNeedsInnerSize(utb: UserTypeFromBytes): Boolean = { + val unknownInnerSizeProcess = utb.process match { + case Some(process) => process match { + case ProcessZlib | _: ProcessCustom => true + case _: ProcessXor | _: ProcessRotate => false + } + case None => false + } + val unknownInnerSizePadTerm = utb.bytes match { + case bt: BytesLimitType => + bt.padRight.isDefined || bt.terminator.isDefined + case bt: BytesEosType => + bt.padRight.isDefined || bt.terminator.isDefined + case _ => false + } + unknownInnerSizeProcess || unknownInnerSizePadTerm + } } /** * Generates list of extra attributes required to store intermediate / * virtual stuff for every attribute like: * - * * buffered raw value byte arrays - * * IO objects (?) - * * unprocessed / postprocessed byte arrays + * - buffered raw value byte arrays + * - IO objects (?) + * - unprocessed / postprocessed byte arrays + * - outer and inner sizes of fields with substreams */ object ExtraAttrs { def forClassSpec(curClass: ClassSpec, compiler: ExtraAttrs): List[AttrSpec] = { @@ -51,8 +78,24 @@ object ExtraAttrs { } case utb: UserTypeFromBytes => // User type in a substream + val dynamicSizeAttributes: List[AttrSpec] = if (compiler.config.readWrite) { + val outerSizeOpt: Option[AttrSpec] = if (compiler.writeNeedsOuterSize(utb)) { + Some(AttrSpec(List(), OuterSizeIdentifier(id), CalcIntType, condSpec)) + } else { + None + } + val innerSizeOpt: Option[AttrSpec] = if (compiler.writeNeedsInnerSize(utb)) { + Some(AttrSpec(List(), InnerSizeIdentifier(id), CalcIntType, condSpec)) + } else { + None + } + List(innerSizeOpt, outerSizeOpt).flatten + } else { + List() + } val rawId = RawIdentifier(id) (List(AttrSpec(List(), rawId, utb.bytes, condSpec)) ++ + dynamicSizeAttributes ++ compiler.extraAttrForIO(rawId, condSpec.repeat) ++ forAttr(rawId, utb.bytes, condSpec, compiler)).toList.distinct case st: SwitchType => diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala index 623621cc8..1a9af766e 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala @@ -121,6 +121,7 @@ abstract class LanguageCompiler( def condIfFooter: Unit def condRepeatCommonInit(id: Identifier, dataType: DataType, needRaw: NeedRaw): Unit + def condRepeatCommonWriteInit(id: Identifier, dataType: DataType, needRaw: NeedRaw): Unit = ??? def condRepeatEosHeader(id: Identifier, io: String, dataType: DataType): Unit def condRepeatEosFooter: Unit @@ -139,6 +140,7 @@ abstract class LanguageCompiler( def normalIO: String def useIO(ioEx: Ast.expr): String def pushPos(io: String): Unit + def pushPosForSubIOWriteBackHandler(io: String): Unit = ??? def seek(io: String, pos: Ast.expr): Unit def seekRelative(io: String, relPos: String): Unit = ??? def popPos(io: String): Unit @@ -146,7 +148,7 @@ abstract class LanguageCompiler( def exprIORemainingSize(io: String): String = ??? - def subIOWriteBackHeader(subIO: String, io: String): String = ??? + def subIOWriteBackHeader(subIO: String): String = ??? def subIOWriteBackFooter: Unit = ??? def addChildIO(io: String, childIO: String): Unit = ??? From 175c2785b89d7b8695d126917c6b67fcc05cc59e Mon Sep 17 00:00:00 2001 From: Petr Pucil Date: Fri, 23 Dec 2022 20:30:46 +0100 Subject: [PATCH 61/90] Fix handling `size-eos: true` with `terminator`/`pad-right` --- .../components/EveryWriteIsExpression.scala | 30 +++++++++++-------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala index 9319b9236..db0e6321f 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala @@ -166,15 +166,10 @@ trait EveryWriteIsExpression extends LanguageCompiler with ObjectOrientedLanguag def attrBytesTypeWrite2(io: String, expr: Ast.expr, t: BytesType, exprTypeOpt: Option[DataType]): Unit = t match { - case t: BytesEosType => - attrPrimitiveWrite(io, expr, t, None, exprTypeOpt) - t.terminator.foreach { (term) => - // FIXME: does not take `eos-error: false` into account (assumes `eos-error: true`) - if (!t.include) - attrPrimitiveWrite(io, Ast.expr.IntNum(term), Int1Type(false), None, None) - } - case blt: BytesLimitType => - attrBytesLimitWrite2(io, expr, blt, exprTypeOpt) + case bt: BytesEosType => + attrBytesLimitWrite2(io, expr, bt, exprIORemainingSize(io), bt.padRight, bt.terminator, bt.include, exprTypeOpt) + case bt: BytesLimitType => + attrBytesLimitWrite2(io, expr, bt, expression(bt.size), bt.padRight, bt.terminator, bt.include, exprTypeOpt) case t: BytesTerminatedType => attrPrimitiveWrite(io, expr, t, None, exprTypeOpt) if (!t.include) { @@ -191,12 +186,21 @@ trait EveryWriteIsExpression extends LanguageCompiler with ObjectOrientedLanguag } } - def attrBytesLimitWrite2(io: String, expr: Ast.expr, blt: BytesLimitType, exprTypeOpt: Option[DataType]): Unit = { - val (term, padRight) = (blt.terminator, blt.padRight, blt.include) match { + def attrBytesLimitWrite2( + io: String, + expr: Ast.expr, + bt: BytesType, + sizeExpr: String, + padRight: Option[Int], + terminator: Option[Int], + include: Boolean, + exprTypeOpt: Option[DataType] + ): Unit = { + val (termArg, padRightArg) = (terminator, padRight, include) match { case (None, None, false) => // no terminator, no padding => just a regular output // validation should check that expression's length matches size - attrPrimitiveWrite(io, expr, blt, None, exprTypeOpt) + attrPrimitiveWrite(io, expr, bt, None, exprTypeOpt) return case (_, None, true) => // terminator included, no padding => pad with zeroes @@ -215,7 +219,7 @@ trait EveryWriteIsExpression extends LanguageCompiler with ObjectOrientedLanguag // both terminator and padding specified (t, p) } - attrBytesLimitWrite(io, expr, translator.translate(blt.size), term, padRight) + attrBytesLimitWrite(io, expr, sizeExpr, termArg, padRightArg) } def attrUserTypeWrite( From b5c2e92eb1b041175881fca909d3edbfd93a590f Mon Sep 17 00:00:00 2001 From: Petr Pucil Date: Wed, 28 Dec 2022 23:31:22 +0100 Subject: [PATCH 62/90] Java: call writeBitsInt{Be,Le}(), disable calls to alignToByte() The writeBitsInt{Be,Le}() methods were implemented in: https://github.com/kaitai-io/kaitai_struct_java_runtime/commit/518c53f438e233da4c1be65ed823ac32a899baac {,write}alignToByte() methods are now called inside the runtime library as needed: https://github.com/kaitai-io/kaitai_struct_java_runtime/commit/1bc75aa91199588a1cb12a5a1c672b80b66619ac --- .../io/kaitai/struct/languages/JavaCompiler.scala | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala index 8844829a1..76e373d5d 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala @@ -491,8 +491,10 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def popPos(io: String): Unit = out.puts(s"$io.seek(_pos);") - override def alignToByte(io: String): Unit = - out.puts(s"$io.alignToByte();") + // NOTE: the compiler does not need to output alignToByte() calls for Java anymore, since the byte + // alignment is handled by the runtime library since commit + // https://github.com/kaitai-io/kaitai_struct_java_runtime/commit/1bc75aa91199588a1cb12a5a1c672b80b66619ac + override def alignToByte(io: String): Unit = {} override def attrDebugStart(attrId: Identifier, attrType: DataType, ios: Option[String], rep: RepeatSpec): Unit = { ios.foreach { (io) => @@ -971,9 +973,9 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) case t: ReadableType => s"$io.write${Utils.capitalize(t.apiCall(defEndian))}($expr)" case BitsType1(bitEndian) => - s"$io.writeBitsInt(1, ${translator.boolToInt(valueExpr)})" + s"$io.writeBitsInt${Utils.upperCamelCase(bitEndian.toSuffix)}(1, ${translator.boolToInt(valueExpr)})" case BitsType(width: Int, bitEndian) => - s"$io.writeBitsInt($width, $expr)" + s"$io.writeBitsInt${Utils.upperCamelCase(bitEndian.toSuffix)}($width, $expr)" case _: BytesType => s"$io.writeBytes($expr)" } From bda1842728674c37c3dc2e2703a8414b4f75ed89 Mon Sep 17 00:00:00 2001 From: Petr Pucil Date: Sat, 31 Dec 2022 17:42:42 +0100 Subject: [PATCH 63/90] Move consistency checks from _check() to _write() if needed Also using a common loop for all kinds of repetitions once again (previously, the common loop was used, but in https://github.com/kaitai-io/kaitai_struct_compiler/commit/98978181af5e034690e8d881acf97f270ba9ad0e the loops were made specific for each repetition type). If evaluating a `repeat-expr` or `repeat-until` expression can cause side effects (i.e. when it refers to a parse instance), consistency checks associated with these expressions will be performed in _write() and they will invoke the appropriate parse instances. This ensures that the behavior will be the same as in _read() while the common "foreach" loop is used. --- .../struct/languages/JavaCompiler.scala | 4 - .../components/EveryWriteIsExpression.scala | 125 ++++-- .../languages/components/GenericChecks.scala | 400 +++++++++++++----- 3 files changed, 370 insertions(+), 159 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala index 76e373d5d..75526a995 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala @@ -628,10 +628,6 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts(s"${privateMemberName(id)}.add($tempVar);") } - override def handleAssignmentRepeatUntilIterator(expr: String): Unit = { - out.puts(s"${translator.doName(Identifier.ITERATOR)} = $expr;") - } - override def condRepeatUntilFooter(id: Identifier, io: String, dataType: DataType, untilExpr: expr): Unit = { typeProvider._currentIteratorType = Some(dataType) out.puts("i++;") diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala index db0e6321f..2d16bd867 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala @@ -9,8 +9,19 @@ import io.kaitai.struct.format._ import scala.collection.mutable.ListBuffer import io.kaitai.struct.datatype._ -trait EveryWriteIsExpression extends LanguageCompiler with ObjectOrientedLanguage with EveryReadIsExpression { +trait EveryWriteIsExpression + extends LanguageCompiler + with ObjectOrientedLanguage + with EveryReadIsExpression + with GenericChecks { override def attrWrite(attr: AttrLikeSpec, id: Identifier, defEndian: Option[Endianness]): Unit = { + val checksShouldDependOnIo: Option[Boolean] = + if (userExprDependsOnIo(attr.cond.ifExpr)) { + None + } else { + Some(true) + } + attrParseIfHeader(id, attr.cond.ifExpr) // Manage IO & seeking for ParseInstances @@ -34,13 +45,13 @@ trait EveryWriteIsExpression extends LanguageCompiler with ObjectOrientedLanguag case Some(_: CalcEndian) | Some(InheritedEndian) => // FIXME: rename to indicate that it can be used for both parsing/writing attrParseHybrid( - () => attrWrite0(id, attr, io, Some(LittleEndian)), - () => attrWrite0(id, attr, io, Some(BigEndian)) + () => attrWrite0(id, attr, io, Some(LittleEndian), checksShouldDependOnIo), + () => attrWrite0(id, attr, io, Some(BigEndian), checksShouldDependOnIo) ) case None => - attrWrite0(id, attr, io, None) + attrWrite0(id, attr, io, None, checksShouldDependOnIo) case Some(fe: FixedEndian) => - attrWrite0(id, attr, io, Some(fe)) + attrWrite0(id, attr, io, Some(fe), checksShouldDependOnIo) } attr match { @@ -54,32 +65,35 @@ trait EveryWriteIsExpression extends LanguageCompiler with ObjectOrientedLanguag attrParseIfFooter(attr.cond.ifExpr) } - def attrWrite0(id: Identifier, attr: AttrLikeSpec, io: String, defEndian: Option[FixedEndian]): Unit = { + def attrWrite0( + id: Identifier, + attr: AttrLikeSpec, + io: String, + defEndian: Option[FixedEndian], + checksShouldDependOnIo: Option[Boolean] + ): Unit = { if (attr.cond.repeat != NoRepeat) condRepeatCommonWriteInit(id, attr.dataType, needRaw(attr.dataType)) attr.cond.repeat match { case RepeatEos => - // we could use condRepeatEosHeader instead (we only deal with fixed-size streams when - // writing), but don't have to (there is no difference, because the `repeat: eos` repetition - // doesn't involve user expressions, unlike `repeat: expr` and `repeat: until`) - condRepeatCommonHeader(id, io, attr.dataType) case RepeatExpr(repeatExpr: Ast.expr) => - condRepeatExprHeader(id, io, attr.dataType, repeatExpr) + attrRepeatExprCheck(id, repeatExpr, checksShouldDependOnIo) case RepeatUntil(untilExpr: Ast.expr) => - condRepeatUntilHeader(id, io, attr.dataType, untilExpr) - val expr = itemExpr(id, attr.cond.repeat, false) - handleAssignmentRepeatUntilIterator(translator.translate(expr)) + if (checksShouldDependOnIo.map(shouldDepend => shouldDepend == false).getOrElse(true)) + attrAssertUntilNotEmpty(id) case NoRepeat => } - attrWrite2(id, attr.dataType, io, attr.cond.repeat, false, defEndian) + if (attr.cond.repeat != NoRepeat) { + condRepeatCommonHeader(id, io, attr.dataType) + } + attrWrite2(id, attr.dataType, io, attr.cond.repeat, false, defEndian, checksShouldDependOnIo) attr.cond.repeat match { - case RepeatEos => - condRepeatCommonFooter - case _: RepeatExpr => - condRepeatExprFooter - case RepeatUntil(untilExpr: Ast.expr) => - condRepeatUntilFooter(id, io, attr.dataType, untilExpr) - case NoRepeat => + case repUntil: RepeatUntil => + attrAssertUntilCond(id, attr.dataType, repUntil, false, checksShouldDependOnIo) + case _ => + } + if (attr.cond.repeat != NoRepeat) { + condRepeatCommonFooter } } @@ -90,13 +104,14 @@ trait EveryWriteIsExpression extends LanguageCompiler with ObjectOrientedLanguag rep: RepeatSpec, isRaw: Boolean, defEndian: Option[FixedEndian], + checksShouldDependOnIo: Option[Boolean], exprTypeOpt: Option[DataType] = None ): Unit = { dataType match { case t: UserType => - attrUserTypeWrite(id, t, io, rep, isRaw, defEndian, exprTypeOpt) + attrUserTypeWrite(id, t, io, rep, isRaw, defEndian, checksShouldDependOnIo, exprTypeOpt) case t: BytesType => - attrBytesTypeWrite(id, t, io, rep, isRaw, exprTypeOpt) + attrBytesTypeWrite(id, t, io, rep, isRaw, checksShouldDependOnIo, exprTypeOpt) case st: SwitchType => val isNullable = if (switchBytesOnlyAsRaw) { st.isNullableSwitchRaw @@ -104,9 +119,9 @@ trait EveryWriteIsExpression extends LanguageCompiler with ObjectOrientedLanguag st.isNullable } - attrSwitchTypeWrite(id, st.on, st.cases, io, rep, defEndian, isNullable, st.combinedType) + attrSwitchTypeWrite(id, st.on, st.cases, io, rep, defEndian, checksShouldDependOnIo, isNullable, st.combinedType) case t: StrFromBytesType => - attrStrTypeWrite(id, t, io, rep, isRaw, exprTypeOpt) + attrStrTypeWrite(id, t, io, rep, isRaw, checksShouldDependOnIo, exprTypeOpt) case t: EnumType => val expr = itemExpr(id, rep, isRaw) val exprType = internalEnumIntType(t.basedOn) @@ -117,7 +132,15 @@ trait EveryWriteIsExpression extends LanguageCompiler with ObjectOrientedLanguag } } - def attrBytesTypeWrite(id: Identifier, t: BytesType, io: String, rep: RepeatSpec, isRaw: Boolean, exprTypeOpt: Option[DataType]): Unit = { + def attrBytesTypeWrite( + id: Identifier, + t: BytesType, + io: String, + rep: RepeatSpec, + isRaw: Boolean, + checksShouldDependOnIo: Option[Boolean], + exprTypeOpt: Option[DataType] + ): Unit = { val idToWrite = t.process match { case Some(proc) => val rawId = RawIdentifier(id) @@ -156,15 +179,31 @@ trait EveryWriteIsExpression extends LanguageCompiler with ObjectOrientedLanguag } else { itemExpr(idToWrite, rep, isRaw) } - attrBytesTypeWrite2(io, expr, t, exprTypeOpt) + attrBytesTypeWrite2(id, io, expr, t, checksShouldDependOnIo, exprTypeOpt) } - def attrStrTypeWrite(id: Identifier, t: StrFromBytesType, io: String, rep: RepeatSpec, isRaw: Boolean, exprTypeOpt: Option[DataType]): Unit = { + def attrStrTypeWrite( + id: Identifier, + t: StrFromBytesType, + io: String, + rep: RepeatSpec, + isRaw: Boolean, + checksShouldDependOnIo: Option[Boolean], + exprTypeOpt: Option[DataType] + ): Unit = { val expr = exprStrToBytes(itemExpr(id, rep, isRaw), t.encoding) - attrBytesTypeWrite2(io, expr, t.bytes, exprTypeOpt) + attrBytesTypeWrite2(id, io, expr, t.bytes, checksShouldDependOnIo, exprTypeOpt) } - def attrBytesTypeWrite2(io: String, expr: Ast.expr, t: BytesType, exprTypeOpt: Option[DataType]): Unit = + def attrBytesTypeWrite2( + id: Identifier, + io: String, + expr: Ast.expr, + t: BytesType, + checksShouldDependOnIo: Option[Boolean], + exprTypeOpt: Option[DataType] + ): Unit = { + attrBytesCheck(id, expr, t, checksShouldDependOnIo) t match { case bt: BytesEosType => attrBytesLimitWrite2(io, expr, bt, exprIORemainingSize(io), bt.padRight, bt.terminator, bt.include, exprTypeOpt) @@ -185,6 +224,7 @@ trait EveryWriteIsExpression extends LanguageCompiler with ObjectOrientedLanguag } } } + } def attrBytesLimitWrite2( io: String, @@ -229,6 +269,7 @@ trait EveryWriteIsExpression extends LanguageCompiler with ObjectOrientedLanguag rep: RepeatSpec, isRaw: Boolean, defEndian: Option[FixedEndian], + checksShouldDependOnIo: Option[Boolean], exprTypeOpt: Option[DataType] = None ) = { val exprType = exprTypeOpt.getOrElse(t) @@ -288,7 +329,7 @@ trait EveryWriteIsExpression extends LanguageCompiler with ObjectOrientedLanguag { val parentIO = subIOWriteBackHeader(ioFixed) handleAssignment(rawId, exprStreamToByteArray(ioFixed), rep, true) - attrBytesTypeWrite(rawId, byteType, parentIO, rep, isRaw, exprTypeOpt) + attrBytesTypeWrite(rawId, byteType, parentIO, rep, isRaw, checksShouldDependOnIo, exprTypeOpt) subIOWriteBackFooter } @@ -305,6 +346,7 @@ trait EveryWriteIsExpression extends LanguageCompiler with ObjectOrientedLanguag io: String, rep: RepeatSpec, defEndian: Option[FixedEndian], + checksShouldDependOnIo: Option[Boolean], isNullable: Boolean, assignType: DataType ): Unit = { @@ -315,34 +357,23 @@ trait EveryWriteIsExpression extends LanguageCompiler with ObjectOrientedLanguag (dataType) => { if (isNullable) condIfSetNonNull(id) - attrWrite2(id, dataType, io, rep, false, defEndian, Some(assignType)) + attrWrite2(id, dataType, io, rep, false, defEndian, checksShouldDependOnIo, Some(assignType)) }, (dataType) => if (switchBytesOnlyAsRaw) { dataType match { case t: BytesType => - attrWrite2(RawIdentifier(id), dataType, io, rep, false, defEndian, Some(assignType)) + attrWrite2(RawIdentifier(id), dataType, io, rep, false, defEndian, checksShouldDependOnIo, Some(assignType)) case _ => - attrWrite2(id, dataType, io, rep, false, defEndian, Some(assignType)) + attrWrite2(id, dataType, io, rep, false, defEndian, checksShouldDependOnIo, Some(assignType)) } } else { - attrWrite2(id, dataType, io, rep, false, defEndian, Some(assignType)) + attrWrite2(id, dataType, io, rep, false, defEndian, checksShouldDependOnIo, Some(assignType)) } ) } - def exprStrToBytes(name: Ast.expr, encoding: String) = - Ast.expr.Call( - Ast.expr.Attribute( - name, - Ast.identifier("to_b") - ), - Seq(Ast.expr.Str(encoding)) - ) - def internalEnumIntType(basedOn: IntType): DataType - def handleAssignmentRepeatUntilIterator(expr: String): Unit - def attrPrimitiveWrite(io: String, expr: Ast.expr, dt: DataType, defEndian: Option[FixedEndian], exprTypeOpt: Option[DataType]): Unit def attrBytesLimitWrite(io: String, expr: Ast.expr, size: String, term: Int, padRight: Int): Unit def attrUserTypeInstreamWrite(io: String, expr: Ast.expr, t: DataType, exprType: DataType): Unit diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/GenericChecks.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/GenericChecks.scala index ef5b9e106..003968276 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/GenericChecks.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/GenericChecks.scala @@ -4,8 +4,15 @@ import io.kaitai.struct.datatype.DataType._ import io.kaitai.struct.exprlang.Ast import io.kaitai.struct.format._ -trait GenericChecks extends LanguageCompiler with EveryReadIsExpression with EveryWriteIsExpression { +trait GenericChecks extends LanguageCompiler with EveryReadIsExpression { override def attrCheck(attr: AttrLikeSpec, id: Identifier): Unit = { + val bodyShouldDependOnIo: Option[Boolean] = + if (userExprDependsOnIo(attr.cond.ifExpr)) { + return + } else { + Some(false) + } + attrParseIfHeader(id, attr.cond.ifExpr) val io = normalIO @@ -13,154 +20,315 @@ trait GenericChecks extends LanguageCompiler with EveryReadIsExpression with Eve attr.cond.repeat match { case RepeatEos => condRepeatCommonHeader(id, io, attr.dataType) - attrCheck2(id, attr.dataType, io, attr.cond.repeat, false) + attrCheck2(id, attr.dataType, io, attr.cond.repeat, false, bodyShouldDependOnIo) condRepeatCommonFooter case RepeatExpr(repeatExpr: Ast.expr) => - attrArraySizeCheck(id, repeatExpr) + attrRepeatExprCheck(id, repeatExpr, bodyShouldDependOnIo) condRepeatCommonHeader(id, io, attr.dataType) - attrCheck2(id, attr.dataType, io, attr.cond.repeat, false) + attrCheck2(id, attr.dataType, io, attr.cond.repeat, false, bodyShouldDependOnIo) condRepeatCommonFooter - case rep: RepeatUntil => - // the array must not be empty (always contains at least the `repeat-until: {true}` element) - attrAssertCmp(exprArraySize(Ast.expr.InternalName(id)), Ast.cmpop.Eq, Ast.expr.IntNum(0), idToMsg(id)) + case repUntil: RepeatUntil => + attrAssertUntilNotEmpty(id) condRepeatCommonHeader(id, io, attr.dataType) - attrCheck2(id, attr.dataType, io, attr.cond.repeat, false) - attrAssertUntilCond(id, attr.dataType, rep, false, idToMsg(id)) + attrCheck2(id, attr.dataType, io, attr.cond.repeat, false, bodyShouldDependOnIo) + attrAssertUntilCond(id, attr.dataType, repUntil, false, bodyShouldDependOnIo) condRepeatCommonFooter case NoRepeat => - attrCheck2(id, attr.dataType, io, attr.cond.repeat, false) + attrCheck2(id, attr.dataType, io, attr.cond.repeat, false, bodyShouldDependOnIo) } attrParseIfFooter(attr.cond.ifExpr) } - def attrCheck2(id: Identifier, dataType: DataType, io: String, repeat: RepeatSpec, isRaw: Boolean) = { + + + def userExprDependsOnIo(expr: Option[Ast.expr]): Boolean = expr match { + case None => false + case Some(v) => userExprDependsOnIo(v) + } + + private + def getArrayItemType(dt: DataType): DataType = { + dt match { + case arr: ArrayType => getArrayItemType(arr.elType) + case other => other + } + } + + def userExprDependsOnIo(expr: Ast.expr): Boolean = { + expr match { + case _: Ast.expr.IntNum => false + case _: Ast.expr.FloatNum => false + case _: Ast.expr.Str => false + case _: Ast.expr.Bool => false + case _: Ast.expr.EnumById => false + case _: Ast.expr.EnumByLabel => false + case n: Ast.expr.Name => + val t = getArrayItemType(translator.detectType(n)) + if (t == KaitaiStreamType || t == OwnedKaitaiStreamType) { + true + } else { + /** @see [[ClassTypeProvider.determineType(inClass:ClassSpec,attrName:String):DataType*]] */ + n.id.name match { + case Identifier.ROOT + | Identifier.PARENT + | Identifier.IO + | Identifier.ITERATOR + | Identifier.SWITCH_ON + | Identifier.INDEX + | Identifier.SIZEOF + => false + case _ => + val spec = typeProvider.resolveMember(typeProvider.nowClass, n.id.name) + spec match { + case _: AttrSpec => false + + // Parameters are fine because they are normally set by the user, so are already + // available in _check(). The only parameters set by the generated code in _write() + // (not by the user) are params of type KaitaiStream or an array of KaitaiStream, + // but these were caught earlier in this function. + case _: ParamDefSpec => false + + // Value instances are OK to use in _check() if their expressions in `value` and + // `if` do not use _io or parse instances. They can refer to other value instances, + // provided they follow the same conditions (which is ensured by a recursive call). + case vis: ValueInstanceSpec => + userExprDependsOnIo(vis.ifExpr) || userExprDependsOnIo(vis.value) + + // Although accessing parse instances in _check() is not a problem by itself, + // because parse instances are set by the user so they are already available in + // _check(), it becomes a problem when you don't invoke a parse instance dependent + // on the time of invocation in _write() because you have already done a particular + // check in _check(). + // + // Take the test + // https://github.com/kaitai-io/kaitai_struct_tests/blob/010efd1d9c07a61a320a644d4e782dd488ba28e4/formats/instance_in_repeat_until.ksy + // as an example. In _write() you don't need to reproduce the special `do { ... } + // while (!repeat-until);` loop as used in _read(), because you already know the + // array length, so a simple "foreach" loop will suffice. Then there is a + // consistency check to ensure that the `repeat-until` condition is `false` for all + // items except the last one, and `true` for the last one. This check can be either + // done in _check(), or in _write() at the end of each iteration of the "foreach" + // loop. You can do it in _check() if you want, but you *need* to evaluate the + // `repeat-until` expression (and throw away the result, if you like - the point is + // just to invoke the parse instances specified there) at the end of each "foreach" + // loop iteration in _write(), because _read() does that. So it makes sense to do + // the check only in _write(). + // + // It may be tempting to suggest to do the check both in _check() and _write(), and + // in this particular case you could really do that because the parse instance is + // used directly in the `repeat-until` expression. But if such parse instance is used + // indirectly via a value instance, you should no longer use that value instance in + // _check() at all, because that would cache the its value and the invocation in + // _write() would merely return this cached value, not evaluating the expression + // again. But that means that the parse instance will be written at a different + // time, because it won't be invoked from `seq` at the time it would be in _read() + // and will be written only when invoked from _fetchInstances(), which is wrong and + // inconsistent with parsing. Although the user could work around this specific + // issue by manually invalidating the value instances that the careless _check() + // invoked after calling it, this would be a bug in _check(). Calling _check() + // should not have side effects that the user has to "undo". + // + // Of course, perhaps most parse instances are not dependent on the time of + // invocation. But the language allows them to be, and it's not that trivial to + // detect it: you have to analyze all expressions that affect its parsing. So we + // will not do that for now - it's easier to avoid invoking parse instances in + // _check() (directly or indirectly) entirely. + case _: ParseInstanceSpec => true + } + } + } + case Ast.expr.InternalName(id) => + ??? + case Ast.expr.UnaryOp(op, inner) => + userExprDependsOnIo(inner) + case Ast.expr.Compare(left, op, right) => + userExprDependsOnIo(left) || userExprDependsOnIo(right) + case Ast.expr.BinOp(left, op, right) => + userExprDependsOnIo(left) || userExprDependsOnIo(right) + case Ast.expr.BoolOp(op, values) => + values.exists(v => userExprDependsOnIo(v)) + case Ast.expr.IfExp(condition, ifTrue, ifFalse) => + userExprDependsOnIo(condition) || userExprDependsOnIo(ifTrue) || userExprDependsOnIo(ifFalse) + case Ast.expr.Subscript(value, idx) => + userExprDependsOnIo(value) || userExprDependsOnIo(idx) + case a: Ast.expr.Attribute => + val t = getArrayItemType(translator.detectType(a)) + if (t == KaitaiStreamType || t == OwnedKaitaiStreamType) { + true + } else { + userExprDependsOnIo(a.value) + } + case Ast.expr.Call(func, args) => + (func match { + case Ast.expr.Attribute(value, methodName) => + userExprDependsOnIo(value) + }) || args.exists(v => userExprDependsOnIo(v)) + case Ast.expr.List(values: Seq[Ast.expr]) => + values.exists(v => userExprDependsOnIo(v)) + case Ast.expr.CastToType(value, typeName) => + userExprDependsOnIo(value) + case _: Ast.expr.ByteSizeOfType => false + case _: Ast.expr.BitSizeOfType => false + } + + } + + def attrCheck2(id: Identifier, dataType: DataType, io: String, repeat: RepeatSpec, isRaw: Boolean, shouldDependOnIo: Option[Boolean]) = { val item = itemExpr(id, repeat, isRaw) dataType match { case t: BytesType => - attrBytesCheck(item, t, idToMsg(id)) + attrBytesCheck(id, item, t, shouldDependOnIo) case st: StrFromBytesType => val bytes = exprStrToBytes(item, st.encoding) - attrBytesCheck( - bytes, - st.bytes, - idToMsg(id) - ) + attrBytesCheck(id, bytes, st.bytes, shouldDependOnIo) case _ => // no checks } } - def attrArraySizeCheck(id: Identifier, expectedSize: Ast.expr): Unit = + def attrRepeatExprCheck(id: Identifier, expectedSize: Ast.expr, shouldDependOnIo: Option[Boolean]): Unit = { + if (shouldDependOnIo.map(shouldDepend => userExprDependsOnIo(expectedSize) != shouldDepend).getOrElse(false)) + return attrAssertEqual( exprArraySize(Ast.expr.InternalName(id)), expectedSize, idToMsg(id) ) + } + + def attrBytesCheck(id: Identifier, bytes: Ast.expr, t: BytesType, shouldDependOnIoOrig: Option[Boolean]): Unit = { + val shouldDependOnIo: Option[Boolean] = + if (t.process.isDefined) { + if (shouldDependOnIoOrig.getOrElse(true)) { + None + } else { + return + } + } else { + shouldDependOnIoOrig + } - def attrBytesCheck(bytes: Ast.expr, t: BytesType, msgId: String): Unit = { + val msgId = idToMsg(id) val actualSize = exprByteArraySize(bytes) + val canUseNonIoDependent = shouldDependOnIo.map(shouldDepend => shouldDepend == false).getOrElse(true) t match { case blt: BytesLimitType => { - if (blt.terminator.isDefined || blt.padRight.isDefined) { - // size must be "<= declared" (less than or equal to declared size) - attrAssertLtE(actualSize, blt.size, msgId) - } else { - // size must match declared size exactly - attrAssertEqual( - actualSize, - blt.size, - msgId - ) - } - blt.terminator.foreach { (term) => - val actualIndexOfTerm = exprByteArrayIndexOf(bytes, term) - val isPadRightActive = blt.padRight.map(padByte => padByte != term).getOrElse(false) - if (!blt.include) { - attrAssertEqual(actualIndexOfTerm, Ast.expr.IntNum(-1), msgId) - if (isPadRightActive) { - condIfHeader(Ast.expr.Compare(actualSize, Ast.cmpop.Eq, blt.size)) - // check if the last byte is not `pad-right` - } + val limitSize = blt.size + val canUseLimitSize = shouldDependOnIo.map(shouldDepend => userExprDependsOnIo(limitSize) == shouldDepend).getOrElse(true) + if (canUseLimitSize) { + if (blt.terminator.isDefined || blt.padRight.isDefined) { + // size must be "<= declared" (less than or equal to declared size) + attrAssertLtE(actualSize, limitSize, msgId) } else { - val lastByteIndex = Ast.expr.BinOp(actualSize, Ast.operator.Sub, Ast.expr.IntNum(1)) - if (!isPadRightActive) { - condIfHeader(Ast.expr.Compare(actualSize, Ast.cmpop.Lt, blt.size)) - // must not be empty (always contains at least the `terminator` byte) - attrAssertCmp(actualSize, Ast.cmpop.Eq, Ast.expr.IntNum(0), msgId) - // the user wants to terminate the value prematurely and there's no `pad-right` that - // could do that, so the last byte of the value must be `terminator` - attrAssertEqual(actualIndexOfTerm, lastByteIndex, msgId) - condIfFooter - - condIfHeader(Ast.expr.Compare(actualSize, Ast.cmpop.Eq, blt.size)) - } - attrBasicCheck( - Ast.expr.BoolOp( - Ast.boolop.And, - Seq( - Ast.expr.Compare(actualIndexOfTerm, Ast.cmpop.NotEq, Ast.expr.IntNum(-1)), - Ast.expr.Compare(actualIndexOfTerm, Ast.cmpop.NotEq, lastByteIndex) - ) - ), - actualIndexOfTerm, - lastByteIndex, + // size must match declared size exactly + attrAssertEqual( + actualSize, + limitSize, msgId ) - if (!isPadRightActive) { - condIfFooter - } else { - condIfHeader(Ast.expr.Compare(actualIndexOfTerm, Ast.cmpop.Eq, Ast.expr.IntNum(-1))) - // check if the last byte is not `pad-right` - } } - // intentionally deferring the `condIfFooter` call (in case of `isPadRightActive`) } - blt.padRight.foreach { (padByte) => - if (blt.terminator.map(term => padByte != term).getOrElse(true)) { - val lastByte = exprByteArrayLast(bytes) - attrBasicCheck( - Ast.expr.BoolOp( - Ast.boolop.And, - Seq( - Ast.expr.Compare(actualSize, Ast.cmpop.NotEq, Ast.expr.IntNum(0)), - Ast.expr.Compare( - lastByte, - Ast.cmpop.Eq, - Ast.expr.IntNum(padByte) - ) - ) - ), - lastByte, - Ast.expr.IntNum(padByte), - msgId - ) + blt.terminator match { + case Some(term) => { + val actualIndexOfTerm = exprByteArrayIndexOf(bytes, term) + val isPadRightActive = blt.padRight.map(padByte => padByte != term).getOrElse(false) + if (!blt.include) { + if (canUseNonIoDependent) { + attrAssertEqual(actualIndexOfTerm, Ast.expr.IntNum(-1), msgId) + } + if (isPadRightActive && canUseLimitSize) { + condIfHeader(Ast.expr.Compare(actualSize, Ast.cmpop.Eq, limitSize)) + // check if the last byte is not `pad-right` + attrBytesPadRightCheck(bytes, actualSize, blt.padRight, msgId) + condIfFooter + } + } else { + val lastByteIndex = Ast.expr.BinOp(actualSize, Ast.operator.Sub, Ast.expr.IntNum(1)) + if (!isPadRightActive && canUseLimitSize) { + condIfHeader(Ast.expr.Compare(actualSize, Ast.cmpop.Lt, limitSize)) + // must not be empty (always contains at least the `terminator` byte) + attrAssertCmp(actualSize, Ast.cmpop.Eq, Ast.expr.IntNum(0), msgId) + // the user wants to terminate the value prematurely and there's no `pad-right` that + // could do that, so the last byte of the value must be `terminator` + attrAssertEqual(actualIndexOfTerm, lastByteIndex, msgId) + condIfFooter + + condIfHeader(Ast.expr.Compare(actualSize, Ast.cmpop.Eq, limitSize)) + attrTermIncludeCheck(actualIndexOfTerm, lastByteIndex, msgId) + condIfFooter + } + if (isPadRightActive && canUseNonIoDependent) { + attrTermIncludeCheck(actualIndexOfTerm, lastByteIndex, msgId) + + condIfHeader(Ast.expr.Compare(actualIndexOfTerm, Ast.cmpop.Eq, Ast.expr.IntNum(-1))) + // check if the last byte is not `pad-right` + attrBytesPadRightCheck(bytes, actualSize, blt.padRight, msgId) + condIfFooter + } + } } - } - blt.terminator.foreach { (term) => - val isPadRightActive = blt.padRight.map(padByte => padByte != term).getOrElse(false) - // here's the `condIfFooter` call omitted from the previous `blt.terminator.foreach()` block - if (isPadRightActive) - condIfFooter + case None => + if (canUseLimitSize) { + // check if the last byte is not `pad-right` + attrBytesPadRightCheck(bytes, actualSize, blt.padRight, msgId) + } } } case btt: BytesTerminatedType => { - val actualIndexOfTerm = exprByteArrayIndexOf(bytes, btt.terminator) - // FIXME: does not take `eos-error: false` into account (assumes `eos-error: true`, i.e. the default setting) - val expectedIndexOfTerm = if (btt.include) { - // must not be empty (always contains at least the `terminator` byte) - attrAssertCmp(actualSize, Ast.cmpop.Eq, Ast.expr.IntNum(0), msgId) + if (canUseNonIoDependent) { + val actualIndexOfTerm = exprByteArrayIndexOf(bytes, btt.terminator) + // FIXME: does not take `eos-error: false` into account (assumes `eos-error: true`, i.e. the default setting) + val expectedIndexOfTerm = if (btt.include) { + // must not be empty (always contains at least the `terminator` byte) + attrAssertCmp(actualSize, Ast.cmpop.Eq, Ast.expr.IntNum(0), msgId) - Ast.expr.BinOp(actualSize, Ast.operator.Sub, Ast.expr.IntNum(1)) - } else { - Ast.expr.IntNum(-1) - } + Ast.expr.BinOp(actualSize, Ast.operator.Sub, Ast.expr.IntNum(1)) + } else { + Ast.expr.IntNum(-1) + } - attrAssertEqual(actualIndexOfTerm, expectedIndexOfTerm, msgId) + attrAssertEqual(actualIndexOfTerm, expectedIndexOfTerm, msgId) + } } case _ => // no checks } } + def attrBytesPadRightCheck(bytes: Ast.expr, actualSize: Ast.expr, padRight: Option[Int], msgId: String): Unit = + padRight.foreach { (padByte) => + val lastByte = exprByteArrayLast(bytes) + attrBasicCheck( + Ast.expr.BoolOp( + Ast.boolop.And, + Seq( + Ast.expr.Compare(actualSize, Ast.cmpop.NotEq, Ast.expr.IntNum(0)), + Ast.expr.Compare( + lastByte, + Ast.cmpop.Eq, + Ast.expr.IntNum(padByte) + ) + ) + ), + lastByte, + Ast.expr.IntNum(padByte), + msgId + ) + } + + def attrTermIncludeCheck(actualIndexOfTerm: Ast.expr, lastByteIndex: Ast.expr, msgId: String): Unit = + attrBasicCheck( + Ast.expr.BoolOp( + Ast.boolop.And, + Seq( + Ast.expr.Compare(actualIndexOfTerm, Ast.cmpop.NotEq, Ast.expr.IntNum(-1)), + Ast.expr.Compare(actualIndexOfTerm, Ast.cmpop.NotEq, lastByteIndex) + ) + ), + actualIndexOfTerm, + lastByteIndex, + msgId + ) + def attrBasicCheck(checkExpr: Ast.expr, actual: Ast.expr, expected: Ast.expr, msg: String): Unit private @@ -194,26 +362,42 @@ trait GenericChecks extends LanguageCompiler with EveryReadIsExpression with Eve Seq(Ast.expr.IntNum(term)) ) + def exprStrToBytes(name: Ast.expr, encoding: String) = + Ast.expr.Call( + Ast.expr.Attribute( + name, + Ast.identifier("to_b") + ), + Seq(Ast.expr.Str(encoding)) + ) + def exprArraySize(name: Ast.expr) = exprByteArraySize(name) def exprArrayLast(name: Ast.expr) = exprByteArrayLast(name) - def attrAssertUntilCond(id: Identifier, dataType: DataType, repeat: RepeatUntil, isRaw: Boolean, msg: String): Unit = { + def attrAssertUntilNotEmpty(id: Identifier): Unit = { + // the array must not be empty (always contains at least the `repeat-until: {true}` element) + attrAssertCmp(exprArraySize(Ast.expr.InternalName(id)), Ast.cmpop.Eq, Ast.expr.IntNum(0), idToMsg(id)) + } + + def attrAssertUntilCond(id: Identifier, dataType: DataType, repUntil: RepeatUntil, isRaw: Boolean, shouldDependOnIo: Option[Boolean]): Unit = { + typeProvider._currentIteratorType = Some(dataType) + if (shouldDependOnIo.map(shouldDepend => userExprDependsOnIo(repUntil.expr) != shouldDepend).getOrElse(false)) + return blockScopeHeader handleAssignmentTempVar( dataType, translator.doName(Identifier.ITERATOR), - translator.translate(itemExpr(id, repeat, isRaw)) + translator.translate(itemExpr(id, repUntil, isRaw)) ) - typeProvider._currentIteratorType = Some(dataType) attrAssertEqual( - repeat.expr, + repUntil.expr, Ast.expr.Compare( Ast.expr.Name(Ast.identifier(Identifier.INDEX)), Ast.cmpop.Eq, Ast.expr.BinOp(exprArraySize(Ast.expr.InternalName(id)), Ast.operator.Sub, Ast.expr.IntNum(1)) ), - msg + idToMsg(id) ) blockScopeFooter } From a5dd1f92f0e5b89fc0438cc29b00f12bb33ef5ed Mon Sep 17 00:00:00 2001 From: Petr Pucil Date: Sat, 31 Dec 2022 19:21:26 +0100 Subject: [PATCH 64/90] Add a _write() check on EOF after a `size-eos: true` field --- .../io/kaitai/struct/languages/JavaCompiler.scala | 11 +++++++++++ .../languages/components/EveryWriteIsExpression.scala | 1 + .../struct/languages/components/GenericChecks.scala | 5 +++++ 3 files changed, 17 insertions(+) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala index 75526a995..b02ed94e3 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala @@ -1004,6 +1004,17 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) importList.add("io.kaitai.struct.ConsistencyError") } + override def attrBytesEosCheck(io: String, msg: String): Unit = { + val msgStr = expression(Ast.expr.Str(msg)) + + out.puts(s"if (!($io.isEof()))") + out.inc + out.puts(s"throw new ConsistencyError($msgStr, ${exprIORemainingSize(io)}, 0);") + out.dec + + importList.add("io.kaitai.struct.ConsistencyError") + } + def value2Const(s: String) = Utils.upperUnderscoreCase(s) override def idToStr(id: Identifier): String = JavaCompiler.idToStr(id) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala index 2d16bd867..687ae360b 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala @@ -207,6 +207,7 @@ trait EveryWriteIsExpression t match { case bt: BytesEosType => attrBytesLimitWrite2(io, expr, bt, exprIORemainingSize(io), bt.padRight, bt.terminator, bt.include, exprTypeOpt) + attrBytesEosCheck(id, io) case bt: BytesLimitType => attrBytesLimitWrite2(io, expr, bt, expression(bt.size), bt.padRight, bt.terminator, bt.include, exprTypeOpt) case t: BytesTerminatedType => diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/GenericChecks.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/GenericChecks.scala index 003968276..ec00e49a2 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/GenericChecks.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/GenericChecks.scala @@ -402,6 +402,11 @@ trait GenericChecks extends LanguageCompiler with EveryReadIsExpression { blockScopeFooter } + def attrBytesEosCheck(id: Identifier, io: String): Unit = + attrBytesEosCheck(io, idToMsg(id)) + + def attrBytesEosCheck(io: String, msg: String): Unit + def attrAssertEqual(actual: Ast.expr, expected: Ast.expr, msg: String): Unit = attrAssertCmp(actual, Ast.cmpop.NotEq, expected, msg) From ecb575732a9af3d9c865b9065db91bb27e560106 Mon Sep 17 00:00:00 2001 From: Petr Pucil Date: Sat, 31 Dec 2022 20:17:02 +0100 Subject: [PATCH 65/90] Add a _write() check on EOF states of `repeat: eos` --- .../io/kaitai/struct/languages/JavaCompiler.scala | 10 ++++++++-- .../components/EveryWriteIsExpression.scala | 12 +++++++++++- .../struct/languages/components/GenericChecks.scala | 6 +++--- 3 files changed, 22 insertions(+), 6 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala index b02ed94e3..2436c4ccb 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala @@ -1004,10 +1004,16 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) importList.add("io.kaitai.struct.ConsistencyError") } - override def attrBytesEosCheck(io: String, msg: String): Unit = { + override def attrIsEofCheck(io: String, expectedIsEof: Boolean, msg: String): Unit = { val msgStr = expression(Ast.expr.Str(msg)) - out.puts(s"if (!($io.isEof()))") + val eofExpr = s"$io.isEof()" + val ifExpr = if (expectedIsEof) { + s"!($eofExpr)" + } else { + eofExpr + } + out.puts(s"if ($ifExpr)") out.inc out.puts(s"throw new ConsistencyError($msgStr, ${exprIORemainingSize(io)}, 0);") out.dec diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala index 687ae360b..2eb7fa3fd 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala @@ -86,6 +86,11 @@ trait EveryWriteIsExpression if (attr.cond.repeat != NoRepeat) { condRepeatCommonHeader(id, io, attr.dataType) } + attr.cond.repeat match { + case RepeatEos => + attrIsEofCheck(id, false, io) + case _ => + } attrWrite2(id, attr.dataType, io, attr.cond.repeat, false, defEndian, checksShouldDependOnIo) attr.cond.repeat match { case repUntil: RepeatUntil => @@ -95,6 +100,11 @@ trait EveryWriteIsExpression if (attr.cond.repeat != NoRepeat) { condRepeatCommonFooter } + attr.cond.repeat match { + case RepeatEos => + attrIsEofCheck(id, true, io) + case _ => + } } def attrWrite2( @@ -207,7 +217,7 @@ trait EveryWriteIsExpression t match { case bt: BytesEosType => attrBytesLimitWrite2(io, expr, bt, exprIORemainingSize(io), bt.padRight, bt.terminator, bt.include, exprTypeOpt) - attrBytesEosCheck(id, io) + attrIsEofCheck(id, true, io) case bt: BytesLimitType => attrBytesLimitWrite2(io, expr, bt, expression(bt.size), bt.padRight, bt.terminator, bt.include, exprTypeOpt) case t: BytesTerminatedType => diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/GenericChecks.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/GenericChecks.scala index ec00e49a2..fad88f4cf 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/GenericChecks.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/GenericChecks.scala @@ -402,10 +402,10 @@ trait GenericChecks extends LanguageCompiler with EveryReadIsExpression { blockScopeFooter } - def attrBytesEosCheck(id: Identifier, io: String): Unit = - attrBytesEosCheck(io, idToMsg(id)) + def attrIsEofCheck(id: Identifier, expectedIsEof: Boolean, io: String): Unit = + attrIsEofCheck(io, expectedIsEof, idToMsg(id)) - def attrBytesEosCheck(io: String, msg: String): Unit + def attrIsEofCheck(io: String, expectedIsEof: Boolean, msg: String): Unit def attrAssertEqual(actual: Ast.expr, expected: Ast.expr, msg: String): Unit = attrAssertCmp(actual, Ast.cmpop.NotEq, expected, msg) From 37f432b2033bfabc1178ee022ebcf992767fb6db Mon Sep 17 00:00:00 2001 From: Petr Pucil Date: Sat, 31 Dec 2022 23:49:06 +0100 Subject: [PATCH 66/90] Generate byte array checks for substreams in _write() --- .../struct/languages/components/EveryWriteIsExpression.scala | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala index 2eb7fa3fd..285809a56 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala @@ -119,7 +119,7 @@ trait EveryWriteIsExpression ): Unit = { dataType match { case t: UserType => - attrUserTypeWrite(id, t, io, rep, isRaw, defEndian, checksShouldDependOnIo, exprTypeOpt) + attrUserTypeWrite(id, t, io, rep, isRaw, defEndian, exprTypeOpt) case t: BytesType => attrBytesTypeWrite(id, t, io, rep, isRaw, checksShouldDependOnIo, exprTypeOpt) case st: SwitchType => @@ -280,7 +280,6 @@ trait EveryWriteIsExpression rep: RepeatSpec, isRaw: Boolean, defEndian: Option[FixedEndian], - checksShouldDependOnIo: Option[Boolean], exprTypeOpt: Option[DataType] = None ) = { val exprType = exprTypeOpt.getOrElse(t) @@ -340,7 +339,7 @@ trait EveryWriteIsExpression { val parentIO = subIOWriteBackHeader(ioFixed) handleAssignment(rawId, exprStreamToByteArray(ioFixed), rep, true) - attrBytesTypeWrite(rawId, byteType, parentIO, rep, isRaw, checksShouldDependOnIo, exprTypeOpt) + attrBytesTypeWrite(rawId, byteType, parentIO, rep, isRaw, None, exprTypeOpt) subIOWriteBackFooter } From 4fdf21dd7d637ed1a43734ddad9b8c2eea140bf3 Mon Sep 17 00:00:00 2001 From: Petr Pucil Date: Sun, 1 Jan 2023 11:17:08 +0100 Subject: [PATCH 67/90] Add write support for `eos-error: false` --- .../struct/languages/JavaCompiler.scala | 14 +++++++++ .../components/EveryWriteIsExpression.scala | 31 ++++++++++++++++--- .../languages/components/GenericChecks.scala | 16 +++++----- 3 files changed, 49 insertions(+), 12 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala index 2436c4ccb..fb6959f53 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala @@ -1021,6 +1021,20 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) importList.add("io.kaitai.struct.ConsistencyError") } + override def condIfIsEofHeader(io: String, wantedIsEof: Boolean): Unit = { + val eofExpr = s"$io.isEof()" + val ifExpr = if (!wantedIsEof) { + s"!($eofExpr)" + } else { + eofExpr + } + + out.puts(s"if ($ifExpr) {") + out.inc + } + + override def condIfIsEofFooter: Unit = universalFooter + def value2Const(s: String) = Utils.upperUnderscoreCase(s) override def idToStr(id: Identifier): String = JavaCompiler.idToStr(id) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala index 285809a56..63248ae46 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala @@ -222,17 +222,32 @@ trait EveryWriteIsExpression attrBytesLimitWrite2(io, expr, bt, expression(bt.size), bt.padRight, bt.terminator, bt.include, exprTypeOpt) case t: BytesTerminatedType => attrPrimitiveWrite(io, expr, t, None, exprTypeOpt) - if (!t.include) { + if (t.include) { + val actualIndexOfTerm = exprByteArrayIndexOf(expr, t.terminator) + if (!t.eosError) { + condIfHeader(Ast.expr.Compare(actualIndexOfTerm, Ast.cmpop.Eq, Ast.expr.IntNum(-1))) + attrIsEofCheck(id, true, io) + condIfFooter + } + } else { + if (!t.eosError) + condIfIsEofHeader(io, false) + if (!t.consume) { - blockScopeHeader + if (t.eosError) { + blockScopeHeader + } pushPos(io) } - // FIXME: does not take `eos-error: false` into account (assumes `eos-error: true`) attrPrimitiveWrite(io, Ast.expr.IntNum(t.terminator), Int1Type(false), None, None) if (!t.consume) { popPos(io) - blockScopeFooter + if (t.eosError) { + blockScopeFooter + } } + if (!t.eosError) + condIfIsEofFooter } } } @@ -324,10 +339,13 @@ trait EveryWriteIsExpression seekRelative(io, outerSize) byteType match { case t: BytesTerminatedType => - // FIXME: does not take `eos-error: false` into account (assumes `eos-error: true`) if (!t.include && t.consume) { + if (!t.eosError) + condIfIsEofHeader(io, false) // terminator can only be 1 byte long at the moment seekRelative(io, expression(Ast.expr.IntNum(1))) + if (!t.eosError) + condIfIsEofFooter } case _ => // do nothing } @@ -392,4 +410,7 @@ trait EveryWriteIsExpression def attrUnprocess(proc: ProcessExpr, varSrc: Identifier, varDest: Identifier, rep: RepeatSpec, dt: BytesType, exprTypeOpt: Option[DataType]): Unit def attrUnprocessPrepareBeforeSubIOHandler(proc: ProcessExpr, varSrc: Identifier): Unit + + def condIfIsEofHeader(io: String, wantedIsEof: Boolean): Unit + def condIfIsEofFooter: Unit } diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/GenericChecks.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/GenericChecks.scala index fad88f4cf..f26482a1d 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/GenericChecks.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/GenericChecks.scala @@ -277,17 +277,19 @@ trait GenericChecks extends LanguageCompiler with EveryReadIsExpression { case btt: BytesTerminatedType => { if (canUseNonIoDependent) { val actualIndexOfTerm = exprByteArrayIndexOf(bytes, btt.terminator) - // FIXME: does not take `eos-error: false` into account (assumes `eos-error: true`, i.e. the default setting) + val lastByteIndex: Ast.expr = Ast.expr.BinOp(actualSize, Ast.operator.Sub, Ast.expr.IntNum(1)) val expectedIndexOfTerm = if (btt.include) { - // must not be empty (always contains at least the `terminator` byte) - attrAssertCmp(actualSize, Ast.cmpop.Eq, Ast.expr.IntNum(0), msgId) + if (btt.eosError) { + // must not be empty (always contains at least the `terminator` byte) + attrAssertCmp(actualSize, Ast.cmpop.Eq, Ast.expr.IntNum(0), msgId) - Ast.expr.BinOp(actualSize, Ast.operator.Sub, Ast.expr.IntNum(1)) + attrAssertEqual(actualIndexOfTerm, lastByteIndex, msgId) + } else { + attrTermIncludeCheck(actualIndexOfTerm, lastByteIndex, msgId) + } } else { - Ast.expr.IntNum(-1) + attrAssertEqual(actualIndexOfTerm, Ast.expr.IntNum(-1), msgId) } - - attrAssertEqual(actualIndexOfTerm, expectedIndexOfTerm, msgId) } } case _ => // no checks From a8e4c03a60912e028c611d8939e64f183fa065da Mon Sep 17 00:00:00 2001 From: Petr Pucil Date: Sun, 1 Jan 2023 11:19:22 +0100 Subject: [PATCH 68/90] Convert Identifier to id in ConsistencyError using `humanReadable` --- .../main/scala/io/kaitai/struct/format/Identifier.scala | 2 +- .../struct/languages/components/GenericChecks.scala | 9 +-------- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/format/Identifier.scala b/shared/src/main/scala/io/kaitai/struct/format/Identifier.scala index 92b3e43a4..0b8f74839 100644 --- a/shared/src/main/scala/io/kaitai/struct/format/Identifier.scala +++ b/shared/src/main/scala/io/kaitai/struct/format/Identifier.scala @@ -25,7 +25,7 @@ abstract class Identifier { */ case class NumberedIdentifier(idx: Int) extends Identifier { import NumberedIdentifier._ - override def humanReadable: String = s"${TEMPLATE}_$idx" + override def humanReadable: String = s"_${TEMPLATE}$idx" } object NumberedIdentifier { diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/GenericChecks.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/GenericChecks.scala index f26482a1d..7562ee4a1 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/GenericChecks.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/GenericChecks.scala @@ -334,14 +334,7 @@ trait GenericChecks extends LanguageCompiler with EveryReadIsExpression { def attrBasicCheck(checkExpr: Ast.expr, actual: Ast.expr, expected: Ast.expr, msg: String): Unit private - def idToMsg(id: Identifier): String = id match { - case NumberedIdentifier(idx) => s"_${NumberedIdentifier.TEMPLATE}$idx" - case NamedIdentifier(name) => name - case RawIdentifier(innerId) => s"_raw_$innerId" - case IoStorageIdentifier(innerId) => s"_io_$innerId" - case InstanceIdentifier(name) => name - case SpecialIdentifier(name) => name - } + def idToMsg(id: Identifier): String = id.humanReadable def exprByteArraySize(name: Ast.expr) = Ast.expr.Attribute( From 08ac8097c4d53d954e7e2c8fed52677c9dd76fa9 Mon Sep 17 00:00:00 2001 From: Petr Pucil Date: Sun, 1 Jan 2023 13:48:52 +0100 Subject: [PATCH 69/90] Remove unused `isRaw` parameter in `itemExpr` method --- .../kaitai/struct/languages/JavaCompiler.scala | 2 +- .../components/EveryReadIsExpression.scala | 6 +++--- .../components/EveryWriteIsExpression.scala | 16 ++++++++-------- .../languages/components/FetchInstances.scala | 2 +- .../languages/components/GenericChecks.scala | 4 ++-- 5 files changed, 15 insertions(+), 15 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala index fb6959f53..8e1acab5e 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala @@ -353,7 +353,7 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) // use `_raw_items[_raw_items.size - 1]` case _: RawIdentifier => getRawIdExpr(varSrc, rep) // but `items[_index]` - case _ => expression(itemExpr(varSrc, rep, false)) + case _ => expression(itemExpr(varSrc, rep)) } val srcExpr = castIfNeeded(srcExprRaw, exprType, dataType) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/EveryReadIsExpression.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/EveryReadIsExpression.scala index 53d7c5039..7bf1016f4 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/EveryReadIsExpression.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/EveryReadIsExpression.scala @@ -115,7 +115,7 @@ trait EveryReadIsExpression case None => rawId case Some(_) => RawIdentifier(rawId) } - val item = itemExpr(rawRawId, rep, false) + val item = itemExpr(rawRawId, rep) val itemSizeExprStr = expression(Ast.expr.Attribute(item, Ast.identifier("size"))) /** FIXME: cannot use [[handleAssignment]] because [[handleAssignmentRepeatUntil]] * always tries to assign the value to the [[Identifier.ITERATOR]] variable */ @@ -126,7 +126,7 @@ trait EveryReadIsExpression } } if (writeNeedsInnerSize(knownSizeType)) { - val item = itemExpr(rawId, rep, false) + val item = itemExpr(rawId, rep) val itemSizeExprStr = expression(Ast.expr.Attribute(item, Ast.identifier("size"))) /** FIXME: cannot use [[handleAssignment]] because [[handleAssignmentRepeatUntil]] * always tries to assign the value to the [[Identifier.ITERATOR]] variable */ @@ -231,7 +231,7 @@ trait EveryReadIsExpression handleAssignmentSimple(instName, expression(value)) } - def itemExpr(id: Identifier, rep: RepeatSpec, isRaw: Boolean): Ast.expr = { + def itemExpr(id: Identifier, rep: RepeatSpec): Ast.expr = { val astId = Ast.expr.InternalName(id) rep match { case NoRepeat => diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala index 63248ae46..8688b8de2 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala @@ -133,11 +133,11 @@ trait EveryWriteIsExpression case t: StrFromBytesType => attrStrTypeWrite(id, t, io, rep, isRaw, checksShouldDependOnIo, exprTypeOpt) case t: EnumType => - val expr = itemExpr(id, rep, isRaw) + val expr = itemExpr(id, rep) val exprType = internalEnumIntType(t.basedOn) attrPrimitiveWrite(io, Ast.expr.Attribute(expr, Ast.identifier("to_i")), t.basedOn, defEndian, Some(exprType)) case _ => - val expr = itemExpr(id, rep, isRaw) + val expr = itemExpr(id, rep) attrPrimitiveWrite(io, expr, dataType, defEndian, exprTypeOpt) } } @@ -161,7 +161,7 @@ trait EveryWriteIsExpression } val expr = if (idToWrite.isInstanceOf[RawIdentifier] && rep != NoRepeat) { // NOTE: This special handling isn't normally needed and one can just use - // `itemExpr(idToWrite, rep, isRaw)` as usual. The `itemExpr` method assumes that the + // `itemExpr(idToWrite, rep)` as usual. The `itemExpr` method assumes that the // expression it's supposed to generate will be used in a loop where the iteration // variable `Identifier.INDEX` is available (usually called just `i`) and uses it. This // is a good default, but it doesn't work if the expression is used between @@ -187,7 +187,7 @@ trait EveryWriteIsExpression ) ) } else { - itemExpr(idToWrite, rep, isRaw) + itemExpr(idToWrite, rep) } attrBytesTypeWrite2(id, io, expr, t, checksShouldDependOnIo, exprTypeOpt) } @@ -201,7 +201,7 @@ trait EveryWriteIsExpression checksShouldDependOnIo: Option[Boolean], exprTypeOpt: Option[DataType] ): Unit = { - val expr = exprStrToBytes(itemExpr(id, rep, isRaw), t.encoding) + val expr = exprStrToBytes(itemExpr(id, rep), t.encoding) attrBytesTypeWrite2(id, io, expr, t.bytes, checksShouldDependOnIo, exprTypeOpt) } @@ -298,7 +298,7 @@ trait EveryWriteIsExpression exprTypeOpt: Option[DataType] = None ) = { val exprType = exprTypeOpt.getOrElse(t) - val expr = itemExpr(id, rep, isRaw) + val expr = itemExpr(id, rep) t match { case _: UserTypeInstream => @@ -314,12 +314,12 @@ trait EveryWriteIsExpression case _: BytesEosType => exprIORemainingSize(io) case _: BytesTerminatedType => - translator.translate(itemExpr(OuterSizeIdentifier(id), rep, isRaw)) + translator.translate(itemExpr(OuterSizeIdentifier(id), rep)) } /** @note Must be kept in sync with [[ExtraAttrs.writeNeedsInnerSize]] */ val innerSize = if (writeNeedsInnerSize(utb)) { - translator.translate(itemExpr(InnerSizeIdentifier(id), rep, isRaw)) + translator.translate(itemExpr(InnerSizeIdentifier(id), rep)) } else { outerSize } diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/FetchInstances.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/FetchInstances.scala index 6bf98b092..9f99bf99d 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/FetchInstances.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/FetchInstances.scala @@ -32,7 +32,7 @@ trait FetchInstances extends LanguageCompiler with ObjectOrientedLanguage with E dataType match { case _: UserType => val exprType = exprTypeOpt.getOrElse(dataType) - attrInvokeFetchInstances(itemExpr(id, rep, false), exprType, dataType) + attrInvokeFetchInstances(itemExpr(id, rep), exprType, dataType) case st: SwitchType => attrSwitchTypeFetchInstances(id, st.on, st.cases, rep, st.combinedType) case _ => diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/GenericChecks.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/GenericChecks.scala index 7562ee4a1..70df88e58 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/GenericChecks.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/GenericChecks.scala @@ -175,7 +175,7 @@ trait GenericChecks extends LanguageCompiler with EveryReadIsExpression { } def attrCheck2(id: Identifier, dataType: DataType, io: String, repeat: RepeatSpec, isRaw: Boolean, shouldDependOnIo: Option[Boolean]) = { - val item = itemExpr(id, repeat, isRaw) + val item = itemExpr(id, repeat) dataType match { case t: BytesType => attrBytesCheck(id, item, t, shouldDependOnIo) @@ -383,7 +383,7 @@ trait GenericChecks extends LanguageCompiler with EveryReadIsExpression { handleAssignmentTempVar( dataType, translator.doName(Identifier.ITERATOR), - translator.translate(itemExpr(id, repUntil, isRaw)) + translator.translate(itemExpr(id, repUntil)) ) attrAssertEqual( repUntil.expr, From cf26f3ea394235f49b9c61649747ca79dbb90e27 Mon Sep 17 00:00:00 2001 From: Petr Pucil Date: Sun, 1 Jan 2023 16:01:13 +0100 Subject: [PATCH 70/90] Generate checks for SwitchType as well --- .../components/EveryWriteIsExpression.scala | 17 ++--- .../languages/components/GenericChecks.scala | 67 ++++++++++++++++--- 2 files changed, 67 insertions(+), 17 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala index 8688b8de2..6e0a5f81f 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala @@ -94,7 +94,7 @@ trait EveryWriteIsExpression attrWrite2(id, attr.dataType, io, attr.cond.repeat, false, defEndian, checksShouldDependOnIo) attr.cond.repeat match { case repUntil: RepeatUntil => - attrAssertUntilCond(id, attr.dataType, repUntil, false, checksShouldDependOnIo) + attrAssertUntilCond(id, attr.dataType, repUntil, checksShouldDependOnIo) case _ => } if (attr.cond.repeat != NoRepeat) { @@ -129,7 +129,7 @@ trait EveryWriteIsExpression st.isNullable } - attrSwitchTypeWrite(id, st.on, st.cases, io, rep, defEndian, checksShouldDependOnIo, isNullable, st.combinedType) + attrSwitchTypeWrite(id, st.on, st.cases, io, rep, defEndian, checksShouldDependOnIo, st.combinedType) case t: StrFromBytesType => attrStrTypeWrite(id, t, io, rep, isRaw, checksShouldDependOnIo, exprTypeOpt) case t: EnumType => @@ -374,17 +374,18 @@ trait EveryWriteIsExpression io: String, rep: RepeatSpec, defEndian: Option[FixedEndian], - checksShouldDependOnIo: Option[Boolean], - isNullable: Boolean, + checksShouldDependOnIoOrig: Option[Boolean], assignType: DataType ): Unit = { - if (isNullable) - condIfSetNull(id) + val checksShouldDependOnIo = + if (userExprDependsOnIo(on)) { + None + } else { + checksShouldDependOnIoOrig + } switchCases[DataType](id, on, cases, (dataType) => { - if (isNullable) - condIfSetNonNull(id) attrWrite2(id, dataType, io, rep, false, defEndian, checksShouldDependOnIo, Some(assignType)) }, (dataType) => if (switchBytesOnlyAsRaw) { diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/GenericChecks.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/GenericChecks.scala index 70df88e58..9c425f3dd 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/GenericChecks.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/GenericChecks.scala @@ -20,21 +20,21 @@ trait GenericChecks extends LanguageCompiler with EveryReadIsExpression { attr.cond.repeat match { case RepeatEos => condRepeatCommonHeader(id, io, attr.dataType) - attrCheck2(id, attr.dataType, io, attr.cond.repeat, false, bodyShouldDependOnIo) + attrCheck2(id, attr.dataType, attr.cond.repeat, bodyShouldDependOnIo) condRepeatCommonFooter case RepeatExpr(repeatExpr: Ast.expr) => attrRepeatExprCheck(id, repeatExpr, bodyShouldDependOnIo) condRepeatCommonHeader(id, io, attr.dataType) - attrCheck2(id, attr.dataType, io, attr.cond.repeat, false, bodyShouldDependOnIo) + attrCheck2(id, attr.dataType, attr.cond.repeat, bodyShouldDependOnIo) condRepeatCommonFooter case repUntil: RepeatUntil => attrAssertUntilNotEmpty(id) condRepeatCommonHeader(id, io, attr.dataType) - attrCheck2(id, attr.dataType, io, attr.cond.repeat, false, bodyShouldDependOnIo) - attrAssertUntilCond(id, attr.dataType, repUntil, false, bodyShouldDependOnIo) + attrCheck2(id, attr.dataType, attr.cond.repeat, bodyShouldDependOnIo) + attrAssertUntilCond(id, attr.dataType, repUntil, bodyShouldDependOnIo) condRepeatCommonFooter case NoRepeat => - attrCheck2(id, attr.dataType, io, attr.cond.repeat, false, bodyShouldDependOnIo) + attrCheck2(id, attr.dataType, attr.cond.repeat, bodyShouldDependOnIo) } attrParseIfFooter(attr.cond.ifExpr) @@ -174,14 +174,26 @@ trait GenericChecks extends LanguageCompiler with EveryReadIsExpression { } - def attrCheck2(id: Identifier, dataType: DataType, io: String, repeat: RepeatSpec, isRaw: Boolean, shouldDependOnIo: Option[Boolean]) = { + def attrCheck2(id: Identifier, dataType: DataType, repeat: RepeatSpec, shouldDependOnIo: Option[Boolean], exprTypeOpt: Option[DataType] = None) = { val item = itemExpr(id, repeat) dataType match { case t: BytesType => - attrBytesCheck(id, item, t, shouldDependOnIo) + val itemBytes = + if (exprTypeOpt.map(exprType => !exprType.isInstanceOf[BytesType]).getOrElse(false)) + Ast.expr.CastToType(item, Ast.typeId(false, Seq("bytes"))) + else + item + attrBytesCheck(id, itemBytes, t, shouldDependOnIo) case st: StrFromBytesType => - val bytes = exprStrToBytes(item, st.encoding) + val itemStr = + if (exprTypeOpt.map(exprType => !exprType.isInstanceOf[StrType]).getOrElse(false)) + Ast.expr.CastToType(item, Ast.typeId(false, Seq("str"))) + else + item + val bytes = exprStrToBytes(itemStr, st.encoding) attrBytesCheck(id, bytes, st.bytes, shouldDependOnIo) + case st: SwitchType => + attrSwitchCheck(id, st.on, st.cases, repeat, shouldDependOnIo, st.combinedType) case _ => // no checks } } @@ -331,6 +343,43 @@ trait GenericChecks extends LanguageCompiler with EveryReadIsExpression { msgId ) + + def attrSwitchCheck( + id: Identifier, + on: Ast.expr, + cases: Map[Ast.expr, DataType], + rep: RepeatSpec, + shouldDependOnIoOrig: Option[Boolean], + assignType: DataType + ): Unit = { + val shouldDependOnIo: Option[Boolean] = + if (userExprDependsOnIo(on)) { + if (shouldDependOnIoOrig.getOrElse(true)) { + None + } else { + return + } + } else { + shouldDependOnIoOrig + } + + switchCases[DataType](id, on, cases, + (dataType) => { + attrCheck2(id, dataType, rep, shouldDependOnIo, Some(assignType)) + }, + (dataType) => if (switchBytesOnlyAsRaw) { + dataType match { + case t: BytesType => + attrCheck2(RawIdentifier(id), dataType, rep, shouldDependOnIo, Some(assignType)) + case _ => + attrCheck2(id, dataType, rep, shouldDependOnIo, Some(assignType)) + } + } else { + attrCheck2(id, dataType, rep, shouldDependOnIo, Some(assignType)) + } + ) + } + def attrBasicCheck(checkExpr: Ast.expr, actual: Ast.expr, expected: Ast.expr, msg: String): Unit private @@ -375,7 +424,7 @@ trait GenericChecks extends LanguageCompiler with EveryReadIsExpression { attrAssertCmp(exprArraySize(Ast.expr.InternalName(id)), Ast.cmpop.Eq, Ast.expr.IntNum(0), idToMsg(id)) } - def attrAssertUntilCond(id: Identifier, dataType: DataType, repUntil: RepeatUntil, isRaw: Boolean, shouldDependOnIo: Option[Boolean]): Unit = { + def attrAssertUntilCond(id: Identifier, dataType: DataType, repUntil: RepeatUntil, shouldDependOnIo: Option[Boolean]): Unit = { typeProvider._currentIteratorType = Some(dataType) if (shouldDependOnIo.map(shouldDepend => userExprDependsOnIo(repUntil.expr) != shouldDepend).getOrElse(false)) return From 37da6ddd0903b91d2f0d695956989b6fe8eda2af Mon Sep 17 00:00:00 2001 From: Petr Pucil Date: Sun, 1 Jan 2023 23:10:49 +0100 Subject: [PATCH 71/90] Add checks for user `params`, set `io`/`io[]` params in _write() --- .../struct/languages/JavaCompiler.scala | 16 +++++++ .../components/EveryWriteIsExpression.scala | 22 +++++++++- .../languages/components/GenericChecks.scala | 43 ++++++++++++++++++- 3 files changed, 79 insertions(+), 2 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala index 8e1acab5e..931159c31 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala @@ -288,6 +288,10 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts(s"public void set${idToSetterStr(attrName)}($javaType _v) { $name = _v; }") } + override def attrSetProperty(base: Ast.expr, propName: Identifier, value: String): Unit = { + out.puts(s"${expression(base)}.set${idToSetterStr(propName)}($value);") + } + override def universalDoc(doc: DocSpec): Unit = { out.puts out.puts( "/**") @@ -1004,6 +1008,18 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) importList.add("io.kaitai.struct.ConsistencyError") } + override def attrObjectsEqualCheck(actual: Ast.expr, expected: Ast.expr, msg: String): Unit = { + val msgStr = expression(Ast.expr.Str(msg)) + + out.puts(s"if (!Objects.equals(${expression(actual)}, ${expression(expected)}))") + out.inc + out.puts(s"throw new ConsistencyError($msgStr, ${expression(actual)}, ${expression(expected)});") + out.dec + + importList.add("java.util.Objects") + importList.add("io.kaitai.struct.ConsistencyError") + } + override def attrIsEofCheck(io: String, expectedIsEof: Boolean, msg: String): Unit = { val msgStr = expression(Ast.expr.Str(msg)) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala index 6e0a5f81f..2b65bf5cc 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala @@ -119,7 +119,7 @@ trait EveryWriteIsExpression ): Unit = { dataType match { case t: UserType => - attrUserTypeWrite(id, t, io, rep, isRaw, defEndian, exprTypeOpt) + attrUserTypeWrite(id, t, io, rep, isRaw, defEndian, checksShouldDependOnIo, exprTypeOpt) case t: BytesType => attrBytesTypeWrite(id, t, io, rep, isRaw, checksShouldDependOnIo, exprTypeOpt) case st: SwitchType => @@ -295,11 +295,29 @@ trait EveryWriteIsExpression rep: RepeatSpec, isRaw: Boolean, defEndian: Option[FixedEndian], + checksShouldDependOnIo: Option[Boolean], exprTypeOpt: Option[DataType] = None ) = { val exprType = exprTypeOpt.getOrElse(t) val expr = itemExpr(id, rep) + { + val itemUserType = + if (exprTypeOpt.map(exprType => !exprType.isInstanceOf[UserType]).getOrElse(false)) + Ast.expr.CastToType(expr, Ast.typeId(true, t.classSpec.get.name)) + else + expr + // check non-`io` params + attrUserTypeCheck(id, itemUserType, t, checksShouldDependOnIo) + // set `io` params + (t.classSpec.get.params, t.args).zipped.foreach { (paramDef, argExpr) => + val paramItemType = getArrayItemType(paramDef.dataType) + val paramBasedOnIo = (paramItemType == KaitaiStreamType || paramItemType == OwnedKaitaiStreamType) + if (paramBasedOnIo) + attrSetProperty(itemUserType, paramDef.id, expression(argExpr)) + } + } + t match { case _: UserTypeInstream => attrUserTypeInstreamWrite(io, expr, t, exprType) @@ -414,4 +432,6 @@ trait EveryWriteIsExpression def condIfIsEofHeader(io: String, wantedIsEof: Boolean): Unit def condIfIsEofFooter: Unit + + def attrSetProperty(base: Ast.expr, propName: Identifier, value: String): Unit } diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/GenericChecks.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/GenericChecks.scala index 9c425f3dd..003802d97 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/GenericChecks.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/GenericChecks.scala @@ -47,7 +47,6 @@ trait GenericChecks extends LanguageCompiler with EveryReadIsExpression { case Some(v) => userExprDependsOnIo(v) } - private def getArrayItemType(dt: DataType): DataType = { dt match { case arr: ArrayType => getArrayItemType(arr.elType) @@ -177,6 +176,13 @@ trait GenericChecks extends LanguageCompiler with EveryReadIsExpression { def attrCheck2(id: Identifier, dataType: DataType, repeat: RepeatSpec, shouldDependOnIo: Option[Boolean], exprTypeOpt: Option[DataType] = None) = { val item = itemExpr(id, repeat) dataType match { + case ut: UserType => + val itemUserType = + if (exprTypeOpt.map(exprType => !exprType.isInstanceOf[UserType]).getOrElse(false)) + Ast.expr.CastToType(item, Ast.typeId(true, ut.classSpec.get.name)) + else + item + attrUserTypeCheck(id, itemUserType, ut, shouldDependOnIo) case t: BytesType => val itemBytes = if (exprTypeOpt.map(exprType => !exprType.isInstanceOf[BytesType]).getOrElse(false)) @@ -343,6 +349,38 @@ trait GenericChecks extends LanguageCompiler with EveryReadIsExpression { msgId ) + def attrUserTypeCheck(id: Identifier, utExpr: Ast.expr, ut: UserType, shouldDependOnIo: Option[Boolean]): Unit = { + (ut.classSpec.get.params, ut.args).zipped.foreach { (paramDef, argExpr) => + attrUserTypeParamCheck(id, ut, utExpr, paramDef.id, paramDef.dataType, argExpr, shouldDependOnIo) + } + } + + def attrUserTypeParamCheck(id: Identifier, ut: UserType, utExpr: Ast.expr, paramId: Identifier, paramDataType: DataType, argExpr: Ast.expr, shouldDependOnIo: Option[Boolean]): Unit = { + val paramItemType = getArrayItemType(paramDataType) + val paramBasedOnIo = (paramItemType == KaitaiStreamType || paramItemType == OwnedKaitaiStreamType) + // parameters with types `io` or `io[]` never have to be checked for consistency because they're set by the generated code + if (paramBasedOnIo) + return + if (shouldDependOnIo.map(shouldDepend => userExprDependsOnIo(argExpr) != shouldDepend).getOrElse(false)) + return + val paramAttrName = paramId match { + case NamedIdentifier(name) => name + case SpecialIdentifier(name) => name + } + val actualArgExpr = Ast.expr.Attribute(utExpr, Ast.identifier(paramAttrName)) + val msgId = idToMsg(id) + paramDataType match { + /** @note Must be kept in sync with [[translators.BaseTranslator.translate]] */ + case _: NumericType | _: BooleanType | _: StrType | _: BytesType | _: EnumType => + attrAssertEqual(actualArgExpr, argExpr, msgId) + case _: ArrayType => + attrObjectsEqualCheck(actualArgExpr, argExpr, msgId) + case _: StructType => + attrObjectsEqualCheck(actualArgExpr, argExpr, msgId) + case AnyType => + attrObjectsEqualCheck(actualArgExpr, argExpr, msgId) + } + } def attrSwitchCheck( id: Identifier, @@ -382,6 +420,9 @@ trait GenericChecks extends LanguageCompiler with EveryReadIsExpression { def attrBasicCheck(checkExpr: Ast.expr, actual: Ast.expr, expected: Ast.expr, msg: String): Unit + // This may turn out to be too Java-specific method, so we can refactor it later. + def attrObjectsEqualCheck(actual: Ast.expr, expected: Ast.expr, msg: String): Unit + private def idToMsg(id: Identifier): String = id.humanReadable From 2accf38eb010a242b73aaf65f9ed0a213778b9b0 Mon Sep 17 00:00:00 2001 From: Petr Pucil Date: Mon, 2 Jan 2023 14:12:45 +0100 Subject: [PATCH 72/90] Add checks for `_root` and `_parent` built-in parameters --- .../struct/languages/JavaCompiler.scala | 24 +++++++++++++++++++ .../languages/components/GenericChecks.scala | 10 ++++++++ 2 files changed, 34 insertions(+) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala index 931159c31..fd347b63f 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala @@ -1020,6 +1020,30 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) importList.add("io.kaitai.struct.ConsistencyError") } + override def attrParentParamCheck(actualParentExpr: Ast.expr, ut: UserType, shouldDependOnIo: Option[Boolean], msg: String): Unit = { + if (ut.isOpaque) + return + /** @note Must be kept in sync with [[JavaCompiler.parseExpr]] */ + val (expectedParent, dependsOnIo) = ut.forcedParent match { + case Some(USER_TYPE_NO_PARENT) => ("null", false) + case Some(fp) => + (expression(fp), userExprDependsOnIo(fp)) + case None => ("this", false) + } + if (shouldDependOnIo.map(shouldDepend => dependsOnIo != shouldDepend).getOrElse(false)) + return + + val msgStr = expression(Ast.expr.Str(msg)) + + out.puts(s"if (!Objects.equals(${expression(actualParentExpr)}, $expectedParent))") + out.inc + out.puts(s"throw new ConsistencyError($msgStr, ${expression(actualParentExpr)}, $expectedParent);") + out.dec + + importList.add("java.util.Objects") + importList.add("io.kaitai.struct.ConsistencyError") + } + override def attrIsEofCheck(io: String, expectedIsEof: Boolean, msg: String): Unit = { val msgStr = expression(Ast.expr.Str(msg)) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/GenericChecks.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/GenericChecks.scala index 003802d97..2ff21c753 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/GenericChecks.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/GenericChecks.scala @@ -350,6 +350,11 @@ trait GenericChecks extends LanguageCompiler with EveryReadIsExpression { ) def attrUserTypeCheck(id: Identifier, utExpr: Ast.expr, ut: UserType, shouldDependOnIo: Option[Boolean]): Unit = { + /** @note Must be kept in sync with [[JavaCompiler.parseExpr]] */ + if (!ut.isOpaque) { + attrUserTypeParamCheck(id, ut, utExpr, RootIdentifier, CalcKaitaiStructType, Ast.expr.Name(Ast.identifier(Identifier.ROOT)), shouldDependOnIo) + } + attrParentParamCheck(id, Ast.expr.Attribute(utExpr, Ast.identifier(Identifier.PARENT)), ut, shouldDependOnIo) (ut.classSpec.get.params, ut.args).zipped.foreach { (paramDef, argExpr) => attrUserTypeParamCheck(id, ut, utExpr, paramDef.id, paramDef.dataType, argExpr, shouldDependOnIo) } @@ -492,6 +497,11 @@ trait GenericChecks extends LanguageCompiler with EveryReadIsExpression { def attrIsEofCheck(io: String, expectedIsEof: Boolean, msg: String): Unit + def attrParentParamCheck(id: Identifier, actualParentExpr: Ast.expr, ut: UserType, shouldDependOnIo: Option[Boolean]): Unit = + attrParentParamCheck(actualParentExpr, ut, shouldDependOnIo, idToMsg(id)) + + def attrParentParamCheck(actualParentExpr: Ast.expr, ut: UserType, shouldDependOnIo: Option[Boolean], msg: String): Unit + def attrAssertEqual(actual: Ast.expr, expected: Ast.expr, msg: String): Unit = attrAssertCmp(actual, Ast.cmpop.NotEq, expected, msg) From 5afa962d24c514d4c1edd559d7d47e461ff30831 Mon Sep 17 00:00:00 2001 From: Petr Pucil Date: Mon, 2 Jan 2023 14:58:08 +0100 Subject: [PATCH 73/90] Add `valid` checks as in _read() to _check()/_write() --- .../languages/components/GenericChecks.scala | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/GenericChecks.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/GenericChecks.scala index 2ff21c753..1d9e90e70 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/GenericChecks.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/GenericChecks.scala @@ -36,6 +36,14 @@ trait GenericChecks extends LanguageCompiler with EveryReadIsExpression { case NoRepeat => attrCheck2(id, attr.dataType, attr.cond.repeat, bodyShouldDependOnIo) } + // TODO: move to attrCheck2 when we change the `valid` semantics to apply to each item + // in repeated fields, not the entire array (see + // https://github.com/kaitai-io/kaitai_struct_formats/issues/347) + attr.valid.foreach { (valid) => + if (bodyShouldDependOnIo.map(shouldDepend => validDependsOnIo(valid) == shouldDepend).getOrElse(true)) { + attrValidate(id, attr, valid) + } + } attrParseIfFooter(attr.cond.ifExpr) } @@ -170,7 +178,17 @@ trait GenericChecks extends LanguageCompiler with EveryReadIsExpression { case _: Ast.expr.ByteSizeOfType => false case _: Ast.expr.BitSizeOfType => false } + } + def validDependsOnIo(valid: ValidationSpec): Boolean = { + valid match { + case ValidationEq(expected) => userExprDependsOnIo(expected) + case ValidationMin(min) => userExprDependsOnIo(min) + case ValidationMax(max) => userExprDependsOnIo(max) + case ValidationRange(min, max) => userExprDependsOnIo(min) || userExprDependsOnIo(max) + case ValidationAnyOf(values) => values.exists(v => userExprDependsOnIo(v)) + case ValidationExpr(expr) => userExprDependsOnIo(expr) + } } def attrCheck2(id: Identifier, dataType: DataType, repeat: RepeatSpec, shouldDependOnIo: Option[Boolean], exprTypeOpt: Option[DataType] = None) = { From 793f58b99945f80bf51bdc85db64123477bc382e Mon Sep 17 00:00:00 2001 From: Petr Pucil Date: Tue, 3 Jan 2023 23:30:11 +0100 Subject: [PATCH 74/90] Add necessary fixes after testing on https://github.com/kaitai-io/kaitai_struct_formats --- .../components/EveryWriteIsExpression.scala | 19 +++++++++++++++---- .../languages/components/GenericChecks.scala | 1 + .../struct/translators/JavaTranslator.scala | 3 +++ 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala index 2b65bf5cc..39cf455e3 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala @@ -159,7 +159,7 @@ trait EveryWriteIsExpression case None => id } - val expr = if (idToWrite.isInstanceOf[RawIdentifier] && rep != NoRepeat) { + val item = if (idToWrite.isInstanceOf[RawIdentifier] && rep != NoRepeat) { // NOTE: This special handling isn't normally needed and one can just use // `itemExpr(idToWrite, rep)` as usual. The `itemExpr` method assumes that the // expression it's supposed to generate will be used in a loop where the iteration @@ -189,7 +189,12 @@ trait EveryWriteIsExpression } else { itemExpr(idToWrite, rep) } - attrBytesTypeWrite2(id, io, expr, t, checksShouldDependOnIo, exprTypeOpt) + val itemBytes = + if (exprTypeOpt.map(exprType => !exprType.isInstanceOf[BytesType]).getOrElse(false)) + Ast.expr.CastToType(item, Ast.typeId(false, Seq("bytes"))) + else + item + attrBytesTypeWrite2(id, io, itemBytes, t, checksShouldDependOnIo, exprTypeOpt) } def attrStrTypeWrite( @@ -201,8 +206,14 @@ trait EveryWriteIsExpression checksShouldDependOnIo: Option[Boolean], exprTypeOpt: Option[DataType] ): Unit = { - val expr = exprStrToBytes(itemExpr(id, rep), t.encoding) - attrBytesTypeWrite2(id, io, expr, t.bytes, checksShouldDependOnIo, exprTypeOpt) + val item = itemExpr(id, rep) + val itemStr = + if (exprTypeOpt.map(exprType => !exprType.isInstanceOf[StrType]).getOrElse(false)) + Ast.expr.CastToType(item, Ast.typeId(false, Seq("str"))) + else + item + val bytes = exprStrToBytes(itemStr, t.encoding) + attrBytesTypeWrite2(id, io, bytes, t.bytes, checksShouldDependOnIo, exprTypeOpt) } def attrBytesTypeWrite2( diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/GenericChecks.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/GenericChecks.scala index 1d9e90e70..87f5444b1 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/GenericChecks.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/GenericChecks.scala @@ -40,6 +40,7 @@ trait GenericChecks extends LanguageCompiler with EveryReadIsExpression { // in repeated fields, not the entire array (see // https://github.com/kaitai-io/kaitai_struct_formats/issues/347) attr.valid.foreach { (valid) => + typeProvider._currentIteratorType = Some(attr.dataTypeComposite) if (bodyShouldDependOnIo.map(shouldDepend => validDependsOnIo(valid) == shouldDepend).getOrElse(true)) { attrValidate(id, attr, valid) } diff --git a/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala index b5563b786..142f11748 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala @@ -60,6 +60,9 @@ class JavaTranslator(provider: TypeProvider, importList: ImportList, config: Run override def doNumericCompareOp(left: expr, op: cmpop, right: expr): String = s"(${super.doNumericCompareOp(left, op, right)})" + override def doEnumCompareOp(left: expr, op: cmpop, right: expr): String = + s"(${super.doEnumCompareOp(left, op, right)})" + override def doName(s: String) = s match { case Identifier.ITERATOR => "_it" From 533aaa8b08dc660c5304ec3549cc1dadc0da8f7d Mon Sep 17 00:00:00 2001 From: Mikhail Yakshin Date: Mon, 1 Aug 2022 13:04:54 +0100 Subject: [PATCH 75/90] Going forward, starting 0.11-SNAPSHOT --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index 593457999..50ff2117d 100644 --- a/build.sbt +++ b/build.sbt @@ -8,7 +8,7 @@ import sbt.Keys._ resolvers += Resolver.sonatypeRepo("public") val NAME = "kaitai-struct-compiler" -val VERSION = "0.10" +val VERSION = "0.11-SNAPSHOT" val TARGET_LANGS = "C++/STL, C#, Go, Java, JavaScript, Lua, Nim, Perl, PHP, Python, Ruby" val UTF8 = Charset.forName("UTF-8") From 52812f9e90394f167717898d52bad9d290f1beee Mon Sep 17 00:00:00 2001 From: Petr Pucil Date: Tue, 14 Feb 2023 15:44:36 +0100 Subject: [PATCH 76/90] Fix superfluous footer in Lua, Perl, PHP, Python and Ruby These languages use UniversalFooter, which implements fetchInstancesFooter, but fetchInstancesHeader does nothing there - so there was a misplaced footer that did not correspond to any active block, and all generated code was typically broken by that. --- .../main/scala/io/kaitai/struct/languages/JavaCompiler.scala | 2 ++ .../io/kaitai/struct/languages/components/UniversalFooter.scala | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala index fd347b63f..612c01a82 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala @@ -231,6 +231,8 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.inc } + override def fetchInstancesFooter: Unit = universalFooter + override def attrInvokeFetchInstances(baseExpr: Ast.expr, exprType: DataType, dataType: DataType): Unit = { val expr = castIfNeeded(expression(baseExpr), exprType, dataType) out.puts(s"$expr._fetchInstances();") diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/UniversalFooter.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/UniversalFooter.scala index 2f1de52de..2beb55b67 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/UniversalFooter.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/UniversalFooter.scala @@ -16,7 +16,6 @@ trait UniversalFooter extends LanguageCompiler { def classFooter(name: String): Unit = universalFooter def classConstructorFooter: Unit = universalFooter override def readFooter: Unit = universalFooter - override def fetchInstancesFooter: Unit = universalFooter override def writeFooter: Unit = universalFooter override def writeInstanceFooter: Unit = universalFooter override def checkFooter: Unit = universalFooter From 203613b3b6631056f365e144f0c886d6078599e2 Mon Sep 17 00:00:00 2001 From: Petr Pucil Date: Wed, 15 Feb 2023 20:27:02 +0100 Subject: [PATCH 77/90] Delete unused method attrWriteStreamToStream --- .../main/scala/io/kaitai/struct/languages/JavaCompiler.scala | 3 --- .../struct/languages/components/EveryWriteIsExpression.scala | 1 - 2 files changed, 4 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala index 612c01a82..5a8899750 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala @@ -993,9 +993,6 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts(s"$expr._write_Seq($io);") } - override def attrWriteStreamToStream(srcIo: String, dstIo: String) = - out.puts(s"$dstIo.writeStream($srcIo);") - override def exprStreamToByteArray(io: String): String = s"$io.toByteArray()" diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala index 39cf455e3..c3c921cff 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala @@ -435,7 +435,6 @@ trait EveryWriteIsExpression def attrPrimitiveWrite(io: String, expr: Ast.expr, dt: DataType, defEndian: Option[FixedEndian], exprTypeOpt: Option[DataType]): Unit def attrBytesLimitWrite(io: String, expr: Ast.expr, size: String, term: Int, padRight: Int): Unit def attrUserTypeInstreamWrite(io: String, expr: Ast.expr, t: DataType, exprType: DataType): Unit - def attrWriteStreamToStream(srcIo: String, dstIo: String): Unit def exprStreamToByteArray(ioFixed: String): String def attrUnprocess(proc: ProcessExpr, varSrc: Identifier, varDest: Identifier, rep: RepeatSpec, dt: BytesType, exprTypeOpt: Option[DataType]): Unit From d1f16dd847f327e10a05b879e84673cace8db55d Mon Sep 17 00:00:00 2001 From: Petr Pucil Date: Sun, 26 Feb 2023 16:39:40 +0100 Subject: [PATCH 78/90] Port serialization support to Python --- .../io/kaitai/struct/ClassCompiler.scala | 15 +- .../struct/languages/JavaCompiler.scala | 9 +- .../struct/languages/PythonCompiler.scala | 411 +++++++++++++++++- .../components/EveryWriteIsExpression.scala | 4 +- .../components/LanguageCompiler.scala | 7 +- .../struct/translators/PythonTranslator.scala | 16 + 6 files changed, 439 insertions(+), 23 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/ClassCompiler.scala b/shared/src/main/scala/io/kaitai/struct/ClassCompiler.scala index e389454fb..a73e289fa 100644 --- a/shared/src/main/scala/io/kaitai/struct/ClassCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/ClassCompiler.scala @@ -140,7 +140,16 @@ class ClassCompiler( curClass.params ) compileInit(curClass) - curClass.instances.foreach { case (instName, _) => lang.instanceClear(instName) } + if (config.readWrite) { + curClass.instances.foreach { case (instName, instSpec) => + lang.instanceClear(instName) + instSpec match { + case _: ParseInstanceSpec => + lang.instanceWriteFlagInit(instName) + case _: ValueInstanceSpec => // do nothing + } + } + } if (lang.config.autoRead) lang.runRead(curClass.name) lang.classConstructorFooter @@ -272,7 +281,7 @@ class ClassCompiler( case None | Some(_: FixedEndian) => compileSeqWriteProc(seq, instances, None) case Some(CalcEndian(_, _)) | Some(InheritedEndian) => - lang.writeHeader(None) + lang.writeHeader(None, false) lang.runWriteCalc() lang.writeFooter() @@ -321,7 +330,7 @@ class ClassCompiler( } def compileSeqWriteProc(seq: List[AttrSpec], instances: Map[InstanceIdentifier, InstanceSpec], defEndian: Option[FixedEndian]) = { - lang.writeHeader(defEndian) + lang.writeHeader(defEndian, !instances.values.exists(i => i.isInstanceOf[ParseInstanceSpec]) && seq.isEmpty) compileSetInstanceWriteFlags(instances) compileSeqWrite(seq, defEndian) lang.writeFooter() diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala index 5a8899750..a2d8d58a5 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala @@ -42,7 +42,7 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) /** See [[subIOWriteBackHeader]] => the code generated when `true` will be inside the definition * of the "writeBackHandler" callback function. */ - private var inSubIOWriteBackHandler = false; + private var inSubIOWriteBackHandler = false override def universalFooter: Unit = { out.dec @@ -242,7 +242,7 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts(s"${publicMemberName(instName)}();") } - override def writeHeader(endian: Option[FixedEndian]): Unit = { + override def writeHeader(endian: Option[FixedEndian], isEmpty: Boolean): Unit = { out.puts endian match { case Some(e) => @@ -416,7 +416,6 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def allocateIO(varName: Identifier, rep: RepeatSpec): String = { val javaName = idToStr(varName) - val ioName = s"_io_$javaName" val args = rep match { @@ -443,7 +442,7 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def allocateIOGrowing(varName: Identifier): String = allocateIOFixed(varName, "100000") // FIXME to use real growing buffer - override def subIOWriteBackHeader(subIO: String): String = { + override def subIOWriteBackHeader(subIO: String, process: Option[ProcessExpr]): String = { val parentIoName = "parent" out.puts(s"final ${type2class(typeProvider.nowClass.name.last)} _this = this;") out.puts(s"$subIO.setWriteBackHandler(new $kstreamName.WriteBackHandler(_pos2) {") @@ -457,7 +456,7 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) parentIoName } - override def subIOWriteBackFooter: Unit = { + override def subIOWriteBackFooter(subIO: String): Unit = { inSubIOWriteBackHandler = false out.dec diff --git a/shared/src/main/scala/io/kaitai/struct/languages/PythonCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/PythonCompiler.scala index a97c86daa..094f31e77 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/PythonCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/PythonCompiler.scala @@ -16,6 +16,9 @@ class PythonCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) with SingleOutputFile with UniversalFooter with EveryReadIsExpression + with FetchInstances + with EveryWriteIsExpression + with GenericChecks with AllocateIOLocalVar with UniversalDoc with SwitchIfOps @@ -27,6 +30,10 @@ class PythonCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def innerDocstrings = true + /** See [[subIOWriteBackHeader]] => the code generated when `true` will be inside the definition + * of the "write back handler" callback function. */ + private var inSubIOWriteBackHandler = false + override def universalFooter: Unit = { out.dec out.puts @@ -43,7 +50,7 @@ class PythonCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) outHeader.puts importList.add("import kaitaistruct") - importList.add(s"from kaitaistruct import $kstructName, $kstreamName, BytesIO") + importList.add(s"from kaitaistruct import $kstructNameFull, $kstreamName, BytesIO") out.puts out.puts @@ -83,7 +90,7 @@ class PythonCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } override def classHeader(name: String): Unit = { - out.puts(s"class ${type2class(name)}($kstructName):") + out.puts(s"class ${type2class(name)}($kstructNameFull):") out.inc } @@ -91,7 +98,8 @@ class PythonCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) val endianAdd = if (isHybrid) ", _is_le=None" else "" val paramsList = Utils.join(params.map((p) => paramName(p.id)), ", ", ", ", "") - out.puts(s"def __init__(self$paramsList, _io, _parent=None, _root=None$endianAdd):") + val ioDefaultVal = if (config.readWrite) "=None" else "" + out.puts(s"def __init__(self$paramsList, _io$ioDefaultVal, _parent=None, _root=None$endianAdd):") out.inc out.puts("self._io = _io") out.puts("self._parent = _parent") @@ -114,11 +122,11 @@ class PythonCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } override def runReadCalc(): Unit = { - out.puts(s"if not hasattr(self, '_is_le'):") + out.puts("if not hasattr(self, '_is_le'):") out.inc out.puts(s"raise ${ksErrorName(UndecidedEndiannessError)}(" + "\"" + typeProvider.nowClass.path.mkString("/", "/", "") + "\")") out.dec - out.puts(s"elif self._is_le == True:") + out.puts("elif self._is_le == True:") out.inc out.puts("self._read_le()") out.dec @@ -128,6 +136,21 @@ class PythonCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.dec } + override def runWriteCalc(): Unit = { + out.puts("if not hasattr(self, '_is_le'):") + out.inc + out.puts(s"raise ${ksErrorName(UndecidedEndiannessError)}(" + "\"" + typeProvider.nowClass.path.mkString("/", "/", "") + "\")") + out.dec + out.puts("elif self._is_le == True:") + out.inc + out.puts("self._write__seq_le()") + out.dec + out.puts("elif self._is_le == False:") + out.inc + out.puts("self._write__seq_be()") + out.dec + } + override def readHeader(endian: Option[FixedEndian], isEmpty: Boolean): Unit = { val suffix = endian match { case Some(e) => s"_${e.toSuffix}" @@ -139,12 +162,82 @@ class PythonCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts("pass") } - override def readFooter() = universalFooter + override def fetchInstancesHeader(): Unit = { + out.puts + out.puts("def _fetch_instances(self):") + out.inc + out.puts("pass") + } + + override def fetchInstancesFooter: Unit = universalFooter + + override def attrInvokeFetchInstances(baseExpr: Ast.expr, exprType: DataType, dataType: DataType): Unit = { + val expr = expression(baseExpr) + out.puts(s"$expr._fetch_instances()") + } + + override def attrInvokeInstance(instName: InstanceIdentifier): Unit = { + out.puts(s"_ = self.${publicMemberName(instName)}") + } + + override def writeHeader(endian: Option[FixedEndian], isEmpty: Boolean): Unit = { + out.puts + endian match { + case Some(e) => + out.puts(s"def _write__seq_${e.toSuffix}(self):") + out.inc + if (isEmpty) + out.puts("pass") + case None => + out.puts("def _write__seq(self, io=None):") + out.inc + // FIXME: remove super() args when dropping support for Python 2 (see + // https://pylint.readthedocs.io/en/v2.16.2/user_guide/messages/refactor/super-with-arguments.html) + out.puts(s"super(${types2class(typeProvider.nowClass.name)}, self)._write__seq(io)") + } + } + + override def checkHeader(): Unit = { + out.puts + out.puts("def _check(self):") + out.inc + out.puts("pass") + } + + override def writeInstanceHeader(instName: InstanceIdentifier): Unit = { + out.puts + out.puts(s"def _write_${publicMemberName(instName)}(self):") + out.inc + instanceClearWriteFlag(instName) + } + + override def checkInstanceHeader(instName: InstanceIdentifier): Unit = { + out.puts + out.puts(s"def _check_${publicMemberName(instName)}(self):") + out.inc + out.puts("pass") + } override def attributeDeclaration(attrName: Identifier, attrType: DataType, isNullable: Boolean): Unit = {} override def attributeReader(attrName: Identifier, attrType: DataType, isNullable: Boolean): Unit = {} + override def attributeSetter(attrName: Identifier, attrType: DataType, isNullable: Boolean): Unit = { + if (attrName.isInstanceOf[InstanceIdentifier]) { + val name = publicMemberName(attrName) + + out.puts(s"@$name.setter") + out.puts(s"def $name(self, v):") + out.inc + handleAssignmentSimple(attrName, "v") + out.dec + } + } + + override def attrSetProperty(base: Ast.expr, propName: Identifier, value: String): Unit = { + out.puts(s"${expression(base)}.${publicMemberName(propName)} = $value") + } + override def universalDoc(doc: DocSpec): Unit = { val docStr = doc.summary match { case Some(summary) => @@ -215,13 +308,89 @@ class PythonCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) importList.add(s"import $pkgName") s"$pkgName.${type2class(name.last)}" } - out.puts(s"_process = $procClass(${args.map(expression).mkString(", ")})") s"_process.decode($srcExpr)" } handleAssignment(varDest, expr, rep, false) } + override def attrUnprocess(proc: ProcessExpr, varSrc: Identifier, varDest: Identifier, rep: RepeatSpec, dt: BytesType, exprTypeOpt: Option[DataType]): Unit = { + val srcExpr = varSrc match { + // use `_raw_items[_raw_items.size - 1]` + case _: RawIdentifier => getRawIdExpr(varSrc, rep) + // but `items[_index]` + case _ => expression(itemExpr(varSrc, rep)) + } + + val expr = proc match { + case ProcessXor(xorValue) => + val argStr = if (inSubIOWriteBackHandler) "_process_val" else expression(xorValue) + val procName = translator.detectType(xorValue) match { + case _: IntType => "process_xor_one" + case _: BytesType => "process_xor_many" + } + s"$kstreamName.$procName($srcExpr, $argStr)" + case ProcessZlib => + importList.add("import zlib") + s"zlib.compress($srcExpr)" + case ProcessRotate(isLeft, rotValue) => + val argStr = if (inSubIOWriteBackHandler) "_process_val" else expression(rotValue) + val expr = if (!isLeft) { + argStr + } else { + s"8 - ($argStr)" + } + s"$kstreamName.process_rotate_left($srcExpr, $expr, 1)" + case ProcessCustom(name, args) => + val procClass = if (name.length == 1) { + val onlyName = name.head + val className = type2class(onlyName) + importList.add(s"from $onlyName import $className") + className + } else { + val pkgName = name.init.mkString(".") + importList.add(s"import $pkgName") + s"$pkgName.${type2class(name.last)}" + } + + val procName = if (inSubIOWriteBackHandler) { + "_process_val" + } else { + val procName = s"_process_${idToStr(varSrc)}" + out.puts(s"$procName = $procClass(${args.map(expression).mkString(", ")})") + procName + } + s"$procName.encode($srcExpr)" + } + handleAssignment(varDest, expr, rep, false) + } + + override def attrUnprocessPrepareBeforeSubIOHandler(proc: ProcessExpr, varSrc: Identifier): Unit = { + // NOTE: the local variable "_process_val" will be captured in a default value of a parameter + // when defining the "write back handler" function (in subIOWriteBackHeader), see + // https://docs.python.org/3/faq/programming.html#why-do-lambdas-defined-in-a-loop-with-different-values-all-return-the-same-result + proc match { + case ProcessXor(xorValue) => + out.puts(s"_process_val = ${expression(xorValue)}") + case ProcessRotate(_, rotValue) => + out.puts(s"_process_val = ${expression(rotValue)}") + case ProcessZlib => // no process arguments + case ProcessCustom(name, args) => + val procClass = if (name.length == 1) { + val onlyName = name.head + val className = type2class(onlyName) + importList.add(s"from $onlyName import $className") + className + } else { + val pkgName = name.init.mkString(".") + importList.add(s"import $pkgName") + s"$pkgName.${type2class(name.last)}" + } + val procName = "_process_val" + out.puts(s"$procName = $procClass(${args.map(expression).mkString(", ")})") + } + } + override def normalIO: String = "self._io" override def allocateIO(varName: Identifier, rep: RepeatSpec): String = { @@ -234,11 +403,51 @@ class PythonCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) ioName } + override def allocateIOFixed(varName: Identifier, size: String): String = { + val varStr = privateMemberName(varName) + val ioName = s"_io_${idToStr(varName)}" + + out.puts(s"$ioName = $kstreamName(BytesIO(bytes($size)))") + ioName + } + + override def exprIORemainingSize(io: String): String = + s"$io.size() - $io.pos()" + + override def subIOWriteBackHeader(subIO: String, process: Option[ProcessExpr]): String = { + val parentIoName = "parent" + // NOTE: local variables "$subIO" and "_process_val" are captured here as default values of + // "handler" parameters, see + // https://docs.python.org/3/faq/programming.html#why-do-lambdas-defined-in-a-loop-with-different-values-all-return-the-same-result + val processValArg = + process.map(proc => proc match { + case _: ProcessXor | _: ProcessRotate | _: ProcessCustom => + ", _process_val=_process_val" + case _ => + "" + }).getOrElse("") + out.puts(s"def handler(parent, $subIO=$subIO$processValArg):") + out.inc + + inSubIOWriteBackHandler = true + + parentIoName + } + + override def subIOWriteBackFooter(subIO: String): Unit = { + inSubIOWriteBackHandler = false + + out.dec + out.puts(s"$subIO.write_back_handler = $kstreamName.WriteBackHandler(_pos2, handler)") + } + + override def addChildIO(io: String, childIO: String): Unit = + out.puts(s"$io.add_child_stream($childIO)") + def getRawIdExpr(varName: Identifier, rep: RepeatSpec): String = { val memberName = privateMemberName(varName) rep match { case NoRepeat => memberName - case RepeatExpr(_) => s"$memberName[i]" case _ => s"$memberName[-1]" } } @@ -251,9 +460,15 @@ class PythonCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def pushPos(io: String): Unit = out.puts(s"_pos = $io.pos()") + override def pushPosForSubIOWriteBackHandler(io: String): Unit = + out.puts(s"_pos2 = $io.pos()") + override def seek(io: String, pos: Ast.expr): Unit = out.puts(s"$io.seek(${expression(pos)})") + override def seekRelative(io: String, relPos: String): Unit = + out.puts(s"$io.seek($io.pos() + ($relPos))") + override def popPos(io: String): Unit = out.puts(s"$io.seek(_pos)") @@ -299,6 +514,7 @@ class PythonCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def condIfHeader(expr: Ast.expr): Unit = { out.puts(s"if ${expression(expr)}:") out.inc + out.puts("pass") } override def condRepeatCommonInit(id: Identifier, dataType: DataType, needRaw: NeedRaw): Unit = { @@ -306,9 +522,26 @@ class PythonCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts(s"${privateMemberName(RawIdentifier(id))} = []") if (needRaw.level >= 2) out.puts(s"${privateMemberName(RawIdentifier(RawIdentifier(id)))} = []") + if (config.readWrite) { + dataType match { + case utb: UserTypeFromBytes => + if (writeNeedsOuterSize(utb)) + out.puts(s"${privateMemberName(OuterSizeIdentifier(id))} = []") + if (writeNeedsInnerSize(utb)) + out.puts(s"${privateMemberName(InnerSizeIdentifier(id))} = []") + case _ => // do nothing + } + } out.puts(s"${privateMemberName(id)} = []") } + override def condRepeatCommonWriteInit(id: Identifier, dataType: DataType, needRaw: NeedRaw): Unit = { + if (needRaw.level >= 1) + out.puts(s"${privateMemberName(RawIdentifier(id))} = []") + if (needRaw.level >= 2) + out.puts(s"${privateMemberName(RawIdentifier(RawIdentifier(id)))} = []") + } + override def condRepeatEosHeader(id: Identifier, io: String, dataType: DataType): Unit = { out.puts("i = 0") out.puts(s"while not $io.is_eof():") @@ -326,6 +559,16 @@ class PythonCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts(s"for i in range(${expression(repeatExpr)}):") out.inc } + + // used for all repetitions in _check() + override def condRepeatCommonHeader(id: Identifier, io: String, dataType: DataType): Unit = { + // TODO: replace range(len()) with enumerate() (see + // https://pylint.readthedocs.io/en/v2.16.2/user_guide/messages/convention/consider-using-enumerate.html) + out.puts(s"for i in range(len(${privateMemberName(id)})):") + out.inc + out.puts("pass") + } + override def handleAssignmentRepeatExpr(id: Identifier, expr: String): Unit = handleAssignmentRepeatEos(id, expr) @@ -393,8 +636,9 @@ class PythonCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def bytesPadTermExpr(expr0: String, padRight: Option[Int], terminator: Option[Int], include: Boolean) = { val expr1 = padRight match { - case Some(padByte) => s"$kstreamName.bytes_strip_right($expr0, $padByte)" - case None => expr0 + case Some(padByte) if terminator.map(term => padByte != term).getOrElse(true) => + s"$kstreamName.bytes_strip_right($expr0, $padByte)" + case _ => expr0 } val expr2 = terminator match { case Some(term) => s"$kstreamName.bytes_terminate($expr1, $term, ${bool2Py(include)})" @@ -420,11 +664,13 @@ class PythonCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def switchIfCaseFirstStart(condition: Ast.expr): Unit = { out.puts(s"if _on == ${expression(condition)}:") out.inc + out.puts("pass") } override def switchIfCaseStart(condition: Ast.expr): Unit = { out.puts(s"elif _on == ${expression(condition)}:") out.inc + out.puts("pass") } override def switchIfCaseEnd(): Unit = @@ -433,10 +679,28 @@ class PythonCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def switchIfElseStart(): Unit = { out.puts(s"else:") out.inc + out.puts("pass") } override def switchIfEnd(): Unit = {} + override def instanceWriteFlagDeclaration(attrName: InstanceIdentifier): Unit = {} + + override def instanceWriteFlagInit(attrName: InstanceIdentifier): Unit = { + instanceClearWriteFlag(attrName) + out.puts(s"self.${publicMemberName(attrName)}__to_write = True") + } + + override def instanceSetWriteFlag(instName: InstanceIdentifier): Unit = { + out.puts(s"self._should_write_${publicMemberName(instName)} = self.${publicMemberName(instName)}__to_write") + } + + override def instanceClearWriteFlag(instName: InstanceIdentifier): Unit = { + out.puts(s"self._should_write_${publicMemberName(instName)} = False") + } + + override def instanceToWriteSetter(instName: InstanceIdentifier): Unit = {} + override def instanceHeader(className: String, instName: InstanceIdentifier, dataType: DataType, isNullable: Boolean): Unit = { out.puts("@property") out.puts(s"def ${publicMemberName(instName)}(self):") @@ -451,11 +715,25 @@ class PythonCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.puts } + override def instanceCheckWriteFlagAndWrite(instName: InstanceIdentifier): Unit = { + out.puts(s"if self._should_write_${publicMemberName(instName)}:") + out.inc + out.puts(s"self._write_${publicMemberName(instName)}()") + out.dec + } + override def instanceReturn(instName: InstanceIdentifier, attrType: DataType): Unit = { // workaround to avoid Python raising an "AttributeError: instance has no attribute" out.puts(s"return getattr(self, '${idToStr(instName)}', None)") } + override def instanceInvalidate(instName: InstanceIdentifier): Unit = { + out.puts(s"def _invalidate_${publicMemberName(instName)}(self):") + out.inc + out.puts(s"del ${privateMemberName(instName)}") + out.dec + } + override def enumDeclaration(curClass: String, enumName: String, enumColl: Seq[(Long, String)]): Unit = { importList.add("from enum import Enum") @@ -466,6 +744,9 @@ class PythonCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.dec } + override def internalEnumIntType(basedOn: IntType): DataType = + basedOn + override def debugClassSequence(seq: List[AttrSpec]) = { val seqStr = seq.map((attr) => "\"" + idToStr(attr.id) + "\"").mkString(", ") out.puts(s"SEQ_FIELDS = [$seqStr]") @@ -479,6 +760,107 @@ class PythonCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.dec } + override def attrPrimitiveWrite( + io: String, + valueExpr: Ast.expr, + dataType: DataType, + defEndian: Option[FixedEndian], + exprTypeOpt: Option[DataType] + ): Unit = { + val expr = expression(valueExpr) + + val stmt = dataType match { + case t: ReadableType => + s"$io.write_${t.apiCall(defEndian)}($expr)" + case BitsType1(bitEndian) => + s"$io.write_bits_int_${bitEndian.toSuffix}(1, ${translator.boolToInt(valueExpr)})" + case BitsType(width: Int, bitEndian) => + s"$io.write_bits_int_${bitEndian.toSuffix}($width, $expr)" + case _: BytesType => + s"$io.write_bytes($expr)" + } + out.puts(stmt) + } + + override def attrBytesLimitWrite(io: String, expr: Ast.expr, size: String, term: Int, padRight: Int): Unit = + out.puts(s"$io.write_bytes_limit(${expression(expr)}, $size, $term, $padRight)") + + override def attrUserTypeInstreamWrite(io: String, valueExpr: Ast.expr, dataType: DataType, exprType: DataType) = { + val expr = expression(valueExpr) + out.puts(s"$expr._write__seq($io)") + } + + override def exprStreamToByteArray(io: String): String = + s"$io.to_byte_array()" + + override def attrBasicCheck(checkExpr: Ast.expr, actual: Ast.expr, expected: Ast.expr, msg: String): Unit = { + val msgStr = expression(Ast.expr.Str(msg)) + + out.puts(s"if ${expression(checkExpr)}:") + out.inc + out.puts(s"raise kaitaistruct.ConsistencyError($msgStr, ${expression(actual)}, ${expression(expected)})") + out.dec + } + + override def attrObjectsEqualCheck(actual: Ast.expr, expected: Ast.expr, msg: String): Unit = { + val msgStr = expression(Ast.expr.Str(msg)) + + out.puts(s"if ${expression(actual)} != ${expression(expected)}:") + out.inc + out.puts(s"raise kaitaistruct.ConsistencyError($msgStr, ${expression(actual)}, ${expression(expected)})") + out.dec + } + + override def attrParentParamCheck(actualParentExpr: Ast.expr, ut: UserType, shouldDependOnIo: Option[Boolean], msg: String): Unit = { + if (ut.isOpaque) + return + /** @note Must be kept in sync with [[PythonCompiler.parseExpr]] */ + val (expectedParent, dependsOnIo) = ut.forcedParent match { + case Some(USER_TYPE_NO_PARENT) => ("None", false) + case Some(fp) => + (expression(fp), userExprDependsOnIo(fp)) + case None => ("self", false) + } + if (shouldDependOnIo.map(shouldDepend => dependsOnIo != shouldDepend).getOrElse(false)) + return + + val msgStr = expression(Ast.expr.Str(msg)) + + out.puts(s"if ${expression(actualParentExpr)} != $expectedParent:") + out.inc + out.puts(s"raise kaitaistruct.ConsistencyError($msgStr, ${expression(actualParentExpr)}, $expectedParent)") + out.dec + } + + override def attrIsEofCheck(io: String, expectedIsEof: Boolean, msg: String): Unit = { + val msgStr = expression(Ast.expr.Str(msg)) + + val eofExpr = s"$io.is_eof()" + val ifExpr = if (expectedIsEof) { + s"not $eofExpr" + } else { + eofExpr + } + out.puts(s"if $ifExpr:") + out.inc + out.puts(s"raise kaitaistruct.ConsistencyError($msgStr, ${exprIORemainingSize(io)}, 0)") + out.dec + } + + override def condIfIsEofHeader(io: String, wantedIsEof: Boolean): Unit = { + val eofExpr = s"$io.is_eof()" + val ifExpr = if (!wantedIsEof) { + s"not $eofExpr" + } else { + eofExpr + } + + out.puts(s"if $ifExpr:") + out.inc + } + + override def condIfIsEofFooter: Unit = universalFooter + def bool2Py(b: Boolean): String = if (b) { "True" } else { "False" } override def idToStr(id: Identifier): String = PythonCompiler.idToStr(id) @@ -505,6 +887,13 @@ class PythonCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.dec } + def kstructNameFull: String = { + ((config.autoRead, config.readWrite) match { + case (_, true) => "ReadWrite" + case (_, false) => "" + }) + kstructName + } + def userType2class(t: UserType): String = { val name = t.classSpec.get.name val firstName = name.head @@ -533,6 +922,8 @@ object PythonCompiler extends LanguageCompilerStatic case NumberedIdentifier(idx) => s"_${NumberedIdentifier.TEMPLATE}$idx" case InstanceIdentifier(name) => s"_m_$name" case RawIdentifier(innerId) => s"_raw_${idToStr(innerId)}" + case OuterSizeIdentifier(innerId) => s"${idToStr(innerId)}__outer_size" + case InnerSizeIdentifier(innerId) => s"${idToStr(innerId)}__inner_size" } def publicMemberName(id: Identifier): String = diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala index c3c921cff..b27be23c9 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/EveryWriteIsExpression.scala @@ -384,10 +384,10 @@ trait EveryWriteIsExpression } { - val parentIO = subIOWriteBackHeader(ioFixed) + val parentIO = subIOWriteBackHeader(ioFixed, byteType.process) handleAssignment(rawId, exprStreamToByteArray(ioFixed), rep, true) attrBytesTypeWrite(rawId, byteType, parentIO, rep, isRaw, None, exprTypeOpt) - subIOWriteBackFooter + subIOWriteBackFooter(ioFixed) } blockScopeFooter diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala index 1a9af766e..c6fa9e55c 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala @@ -102,7 +102,7 @@ abstract class LanguageCompiler( def attrInvokeFetchInstances(baseExpr: Ast.expr, exprType: DataType, dataType: DataType): Unit = ??? def attrInvokeInstance(instName: InstanceIdentifier): Unit = ??? - def writeHeader(endian: Option[FixedEndian]): Unit = ??? + def writeHeader(endian: Option[FixedEndian], isEmpty: Boolean): Unit = ??? def writeFooter(): Unit = ??? def writeInstanceHeader(instName: InstanceIdentifier): Unit = ??? def writeInstanceFooter(): Unit = ??? @@ -148,8 +148,8 @@ abstract class LanguageCompiler( def exprIORemainingSize(io: String): String = ??? - def subIOWriteBackHeader(subIO: String): String = ??? - def subIOWriteBackFooter: Unit = ??? + def subIOWriteBackHeader(subIO: String, process: Option[ProcessExpr]): String = ??? + def subIOWriteBackFooter(subIO: String): Unit = ??? def addChildIO(io: String, childIO: String): Unit = ??? @@ -158,6 +158,7 @@ abstract class LanguageCompiler( def instanceSetCalculated(instName: InstanceIdentifier): Unit = {} def instanceDeclaration(attrName: InstanceIdentifier, attrType: DataType, isNullable: Boolean): Unit = attributeDeclaration(attrName, attrType, isNullable) def instanceWriteFlagDeclaration(attrName: InstanceIdentifier): Unit = ??? + def instanceWriteFlagInit(attrName: InstanceIdentifier): Unit = {} def instanceSetWriteFlag(instName: InstanceIdentifier): Unit = ??? def instanceClearWriteFlag(instName: InstanceIdentifier): Unit = ??? def instanceToWriteSetter(instName: InstanceIdentifier): Unit = ??? diff --git a/shared/src/main/scala/io/kaitai/struct/translators/PythonTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/PythonTranslator.scala index a75c083a0..c0cc45bf7 100644 --- a/shared/src/main/scala/io/kaitai/struct/translators/PythonTranslator.scala +++ b/shared/src/main/scala/io/kaitai/struct/translators/PythonTranslator.scala @@ -16,6 +16,18 @@ class PythonTranslator(provider: TypeProvider, importList: ImportList) extends B } } + override def doNumericCompareOp(left: Ast.expr, op: Ast.cmpop, right: Ast.expr): String = + s"(${super.doNumericCompareOp(left, op, right)})" + + override def doStrCompareOp(left: Ast.expr, op: Ast.cmpop, right: Ast.expr): String = + s"(${super.doStrCompareOp(left, op, right)})" + + override def doEnumCompareOp(left: Ast.expr, op: Ast.cmpop, right: Ast.expr): String = + s"(${super.doEnumCompareOp(left, op, right)})" + + override def doBytesCompareOp(left: Ast.expr, op: Ast.cmpop, right: Ast.expr): String = + s"(${super.doBytesCompareOp(left, op, right)})" + override def doStringLiteral(s: String): String = "u" + super.doStringLiteral(s) override def doBoolLiteral(n: Boolean): String = if (n) "True" else "False" @@ -104,6 +116,8 @@ class PythonTranslator(provider: TypeProvider, importList: ImportList) extends B } override def bytesToStr(bytesExpr: String, encoding: Ast.expr): String = s"($bytesExpr).decode(${translate(encoding)})" + override def bytesIndexOf(b: Ast.expr, byte: Ast.expr): String = + s"${PythonCompiler.kstreamName}.byte_array_index_of(${translate(b)}, ${translate(byte)})" override def bytesLength(value: Ast.expr): String = s"len(${translate(value)})" @@ -125,6 +139,8 @@ class PythonTranslator(provider: TypeProvider, importList: ImportList) extends B s"(${translate(value)})[::-1]" override def strSubstring(s: Ast.expr, from: Ast.expr, to: Ast.expr): String = s"(${translate(s)})[${translate(from)}:${translate(to)}]" + override def strToBytes(s: Ast.expr, encoding: Ast.expr): String = + s"(${translate(s)}).encode(${translate(encoding)})" override def arrayFirst(a: Ast.expr): String = s"${translate(a)}[0]" From d986731cf72499b7cef219ba6dc9876f3239acfc Mon Sep 17 00:00:00 2001 From: Petr Pucil Date: Sun, 26 Feb 2023 18:22:46 +0100 Subject: [PATCH 79/90] Fix Python 2 compatibility when creating a fixed stream --- .../scala/io/kaitai/struct/languages/PythonCompiler.scala | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/PythonCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/PythonCompiler.scala index 094f31e77..700d68e35 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/PythonCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/PythonCompiler.scala @@ -407,7 +407,9 @@ class PythonCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) val varStr = privateMemberName(varName) val ioName = s"_io_${idToStr(varName)}" - out.puts(s"$ioName = $kstreamName(BytesIO(bytes($size)))") + // NOTE: in Python 2, bytes() converts an integer argument to a string (e.g. bytes(12) => '12'), + // so we have to use bytearray() instead + out.puts(s"$ioName = $kstreamName(BytesIO(bytearray($size)))") ioName } From fca43d9a272dd0f632f4dab097880762b4f00b58 Mon Sep 17 00:00:00 2001 From: Petr Pucil Date: Mon, 6 Mar 2023 19:22:03 +0100 Subject: [PATCH 80/90] Python: disable calls to alignToByte() See https://github.com/kaitai-io/kaitai_struct_compiler/commit/b5c2e92eb1b041175881fca909d3edbfd93a590f {,write}alignToByte() methods are now called inside the runtime library as needed: https://github.com/kaitai-io/kaitai_struct_python_runtime/commit/1cb84b84d358e1cdffe35845d1e6688bff923952 --- .../scala/io/kaitai/struct/languages/PythonCompiler.scala | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/PythonCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/PythonCompiler.scala index 700d68e35..8a478467e 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/PythonCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/PythonCompiler.scala @@ -474,8 +474,10 @@ class PythonCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def popPos(io: String): Unit = out.puts(s"$io.seek(_pos)") - override def alignToByte(io: String): Unit = - out.puts(s"$io.align_to_byte()") + // NOTE: the compiler does not need to output alignToByte() calls for Python anymore, since the byte + // alignment is handled by the runtime library since commit + // https://github.com/kaitai-io/kaitai_struct_python_runtime/commit/1cb84b84d358e1cdffe35845d1e6688bff923952 + override def alignToByte(io: String): Unit = {} override def attrDebugStart(attrId: Identifier, attrType: DataType, ios: Option[String], rep: RepeatSpec): Unit = { ios.foreach { (io) => From e776c982f8ba5b039bdcacd1db48ecbe77588df6 Mon Sep 17 00:00:00 2001 From: Petr Pucil Date: Tue, 7 Mar 2023 15:50:36 +0100 Subject: [PATCH 81/90] Python: use `self` as default `_root` only in top-level types as in Java, see https://github.com/kaitai-io/kaitai_struct_compiler/blob/829a14f1e33e8e48eeae726c8a287a5967bcb668/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala#L153 --- .../scala/io/kaitai/struct/languages/PythonCompiler.scala | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/PythonCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/PythonCompiler.scala index 8a478467e..609bb361e 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/PythonCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/PythonCompiler.scala @@ -103,7 +103,11 @@ class PythonCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) out.inc out.puts("self._io = _io") out.puts("self._parent = _parent") - out.puts("self._root = _root if _root else self") + if (name == rootClassName) { + out.puts("self._root = _root if _root else self") + } else { + out.puts("self._root = _root") + } if (isHybrid) out.puts("self._is_le = _is_le") From 00c831f4e35a9682e5ee07742a6b8cb36958bec5 Mon Sep 17 00:00:00 2001 From: Petr Pucil Date: Thu, 27 Jul 2023 15:56:39 +0200 Subject: [PATCH 82/90] Add "implies --no-auto-read" to --read-write CLI option description --- jvm/src/main/scala/io/kaitai/struct/JavaMain.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jvm/src/main/scala/io/kaitai/struct/JavaMain.scala b/jvm/src/main/scala/io/kaitai/struct/JavaMain.scala index e33de13bf..fa8f0af37 100644 --- a/jvm/src/main/scala/io/kaitai/struct/JavaMain.scala +++ b/jvm/src/main/scala/io/kaitai/struct/JavaMain.scala @@ -59,7 +59,7 @@ object JavaMain { opt[Unit]('w', "read-write") action { (x, c) => c.copy(runtime = c.runtime.copy(readWrite = true, autoRead = false)) - } text("generate read-write support in classes (default: read-only)") + } text("generate read-write support in classes (implies --no-auto-read, default: read-only)") opt[File]('d', "outdir") valueName("") action { (x, c) => c.copy(outDir = x) From f1dc857302bc30a2a259f9e6d7dff12674f451b8 Mon Sep 17 00:00:00 2001 From: Petr Pucil Date: Thu, 27 Jul 2023 16:00:10 +0200 Subject: [PATCH 83/90] Add "Java and Python only" to --read-write CLI option description --- jvm/src/main/scala/io/kaitai/struct/JavaMain.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jvm/src/main/scala/io/kaitai/struct/JavaMain.scala b/jvm/src/main/scala/io/kaitai/struct/JavaMain.scala index fa8f0af37..a271c61cc 100644 --- a/jvm/src/main/scala/io/kaitai/struct/JavaMain.scala +++ b/jvm/src/main/scala/io/kaitai/struct/JavaMain.scala @@ -59,7 +59,7 @@ object JavaMain { opt[Unit]('w', "read-write") action { (x, c) => c.copy(runtime = c.runtime.copy(readWrite = true, autoRead = false)) - } text("generate read-write support in classes (implies --no-auto-read, default: read-only)") + } text("generate read-write support in classes (implies --no-auto-read, Java and Python only, default: read-only)") opt[File]('d', "outdir") valueName("") action { (x, c) => c.copy(outDir = x) From 9142a1bc64a7229041670d784cd00005961140cb Mon Sep 17 00:00:00 2001 From: Petr Pucil Date: Tue, 1 Aug 2023 14:53:21 +0200 Subject: [PATCH 84/90] Improve setting `zeroCopySubstream` to false in `readWrite` mode --- jvm/src/main/scala/io/kaitai/struct/JavaMain.scala | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/jvm/src/main/scala/io/kaitai/struct/JavaMain.scala b/jvm/src/main/scala/io/kaitai/struct/JavaMain.scala index a48a85b6b..0f8cdbded 100644 --- a/jvm/src/main/scala/io/kaitai/struct/JavaMain.scala +++ b/jvm/src/main/scala/io/kaitai/struct/JavaMain.scala @@ -57,9 +57,9 @@ object JavaMain { } } - opt[Unit]('w', "read-write") action { (x, c) => - c.copy(runtime = c.runtime.copy(readWrite = true, autoRead = false, zeroCopySubstream = false)) - } text("generate read-write support in classes (implies --no-auto-read --zero-copy-substream=false, Java and Python only, default: read-only)") + opt[Unit]('w', "read-write") action { (x, c) => + c.copy(runtime = c.runtime.copy(readWrite = true, autoRead = false)) + } text("generate read-write support in classes (implies `--no-auto-read --zero-copy-substream false`, Java and Python only, default: read-only)") opt[File]('d', "outdir") valueName("") action { (x, c) => c.copy(outDir = x) @@ -176,7 +176,13 @@ object JavaMain { version("version") text("output version information and exit") } - parser.parse(args, CLIConfig()) + parser.parse(args, CLIConfig()).map { c => + if (c.runtime.readWrite) { + c.copy(runtime = c.runtime.copy(zeroCopySubstream = false)) + } else { + c + } + } } /** From 2eca3de2c4044b74226d6fcd2f3641a65febd9ab Mon Sep 17 00:00:00 2001 From: Petr Pucil Date: Tue, 1 Aug 2023 21:02:43 +0200 Subject: [PATCH 85/90] Bring back init of instance flags in non-`readWrite` mode (C++, C#) (fix regression from d1f16dd8) --- shared/src/main/scala/io/kaitai/struct/ClassCompiler.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shared/src/main/scala/io/kaitai/struct/ClassCompiler.scala b/shared/src/main/scala/io/kaitai/struct/ClassCompiler.scala index a73e289fa..b3ac0d9e9 100644 --- a/shared/src/main/scala/io/kaitai/struct/ClassCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/ClassCompiler.scala @@ -140,9 +140,9 @@ class ClassCompiler( curClass.params ) compileInit(curClass) + curClass.instances.foreach { case (instName, _) => lang.instanceClear(instName) } if (config.readWrite) { curClass.instances.foreach { case (instName, instSpec) => - lang.instanceClear(instName) instSpec match { case _: ParseInstanceSpec => lang.instanceWriteFlagInit(instName) From a477800c4eba850a00302629289f1ac04f0844d1 Mon Sep 17 00:00:00 2001 From: Petr Pucil Date: Sat, 5 Aug 2023 20:53:31 +0200 Subject: [PATCH 86/90] PythonCompiler: fix "{alignToByte => align_to_byte}()" in comment --- .../main/scala/io/kaitai/struct/languages/JavaCompiler.scala | 4 ++-- .../scala/io/kaitai/struct/languages/PythonCompiler.scala | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala index 5e952dda4..c30936146 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala @@ -494,8 +494,8 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def popPos(io: String): Unit = out.puts(s"$io.seek(_pos);") - // NOTE: the compiler does not need to output alignToByte() calls for Java anymore, since the byte - // alignment is handled by the runtime library since commit + // NOTE: the compiler does not need to output alignToByte() calls for Java anymore, + // since the byte alignment is handled by the runtime library since commit // https://github.com/kaitai-io/kaitai_struct_java_runtime/commit/1bc75aa91199588a1cb12a5a1c672b80b66619ac override def alignToByte(io: String): Unit = {} diff --git a/shared/src/main/scala/io/kaitai/struct/languages/PythonCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/PythonCompiler.scala index 8ec3e69db..1d227dcf1 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/PythonCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/PythonCompiler.scala @@ -483,8 +483,8 @@ class PythonCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def popPos(io: String): Unit = out.puts(s"$io.seek(_pos)") - // NOTE: the compiler does not need to output alignToByte() calls for Python anymore, since the byte - // alignment is handled by the runtime library since commit + // NOTE: the compiler does not need to output align_to_byte() calls for Python anymore, + // since the byte alignment is handled by the runtime library since commit // https://github.com/kaitai-io/kaitai_struct_python_runtime/commit/1cb84b84d358e1cdffe35845d1e6688bff923952 override def alignToByte(io: String): Unit = {} From 8b258e2b74dec1f65af55f1cfe0cce948b8a5ab4 Mon Sep 17 00:00:00 2001 From: Petr Pucil Date: Sat, 5 Aug 2023 21:07:29 +0200 Subject: [PATCH 87/90] Remove alignToByte() insertion logic when writing See https://github.com/kaitai-io/kaitai_struct_compiler/pull/255#discussion_r1285107978 --- shared/src/main/scala/io/kaitai/struct/ClassCompiler.scala | 5 ----- 1 file changed, 5 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/ClassCompiler.scala b/shared/src/main/scala/io/kaitai/struct/ClassCompiler.scala index b3ac0d9e9..90a9524bc 100644 --- a/shared/src/main/scala/io/kaitai/struct/ClassCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/ClassCompiler.scala @@ -369,13 +369,8 @@ class ClassCompiler( } def compileSeqWrite(seq: List[AttrSpec], defEndian: Option[FixedEndian]) = { - var wasUnaligned = false seq.foreach { (attr) => - val nowUnaligned = isUnalignedBits(attr.dataType) - if (wasUnaligned && !nowUnaligned) - lang.alignToByte(lang.normalIO) lang.attrWrite(attr, attr.id, defEndian) - wasUnaligned = nowUnaligned } } From cb0c1ebfa87e7eb8b86756be689e4840cd776544 Mon Sep 17 00:00:00 2001 From: Petr Pucil Date: Mon, 7 Aug 2023 11:20:07 +0200 Subject: [PATCH 88/90] Endianness.fromString(): revert incorrect `case {None => _}` change See https://github.com/kaitai-io/kaitai_struct_compiler/pull/255#discussion_r1285569661 --- .../src/main/scala/io/kaitai/struct/datatype/Endianness.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shared/src/main/scala/io/kaitai/struct/datatype/Endianness.scala b/shared/src/main/scala/io/kaitai/struct/datatype/Endianness.scala index 4f5b4ed24..592c13e07 100644 --- a/shared/src/main/scala/io/kaitai/struct/datatype/Endianness.scala +++ b/shared/src/main/scala/io/kaitai/struct/datatype/Endianness.scala @@ -57,7 +57,7 @@ object Endianness { def fromString(s: Option[String], defaultEndian: Option[Endianness], dt: String, path: List[String]): Option[FixedEndian] = s match { case Some("le") => Some(LittleEndian) case Some("be") => Some(BigEndian) - case _ => + case None => defaultEndian match { case Some(e: FixedEndian) => Some(e) case Some(_: CalcEndian) | Some(InheritedEndian) => None // to be overridden during compile From 0179199cb591e77f6afffe890cb6dd6f998fefb0 Mon Sep 17 00:00:00 2001 From: Petr Pucil Date: Sun, 24 Sep 2023 11:55:49 +0200 Subject: [PATCH 89/90] Stop using `_io` when throwing validation errors from _check() Relevant tests were added in these commits: - Java: https://github.com/kaitai-io/kaitai_struct_tests/commit/e92fb336fb2eda17521224ab05ea416f8052828e - Python: https://github.com/kaitai-io/kaitai_struct_tests/commit/e7869f0ce9a1bd9dc58f5523366615fe9e1fbbc8 This commit is needed for the `testCheckBadValidOldIo` / `test_check_bad_valid_old_io` test methods to pass. The _check() method is intended to verify pure data consistency and is supposed to be called at the time when the actual `_io` is not available yet (or is not in the correct state) and should not be used even if it's not `null`. Before this commit, if we wanted to initialize a KS object by reading an existing stream and then edit the data and write them, _check() would read the position from the old `_io` used for reading and report it in the validation error, which is wrong. --- .../struct/languages/CSharpCompiler.scala | 14 +++-- .../kaitai/struct/languages/CppCompiler.scala | 14 +++-- .../kaitai/struct/languages/GoCompiler.scala | 14 +++-- .../struct/languages/JavaCompiler.scala | 14 +++-- .../struct/languages/JavaScriptCompiler.scala | 14 +++-- .../kaitai/struct/languages/LuaCompiler.scala | 17 +++--- .../kaitai/struct/languages/PHPCompiler.scala | 14 +++-- .../struct/languages/PythonCompiler.scala | 14 +++-- .../struct/languages/RubyCompiler.scala | 14 +++-- .../languages/components/CommonReads.scala | 2 +- .../languages/components/GenericChecks.scala | 2 +- .../languages/components/ValidateOps.scala | 52 ++++++++----------- 12 files changed, 103 insertions(+), 82 deletions(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/CSharpCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/CSharpCompiler.scala index 1e156f127..2487a64e3 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/CSharpCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/CSharpCompiler.scala @@ -580,17 +580,21 @@ class CSharpCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def ksErrorName(err: KSError): String = CSharpCompiler.ksErrorName(err) override def attrValidateExpr( - attrId: Identifier, - attrType: DataType, + attr: AttrLikeSpec, checkExpr: Ast.expr, err: KSError, - errArgs: List[Ast.expr] + useIo: Boolean, + expected: Option[Ast.expr] = None ): Unit = { - val errArgsStr = errArgs.map(translator.translate).mkString(", ") + val errArgsStr = expected.map(expression) ++ List( + expression(Ast.expr.InternalName(attr.id)), + if (useIo) expression(Ast.expr.InternalName(IoIdentifier)) else "null", + expression(Ast.expr.Str(attr.path.mkString("/", "/", ""))) + ) out.puts(s"if (!(${translator.translate(checkExpr)}))") out.puts("{") out.inc - out.puts(s"throw new ${ksErrorName(err)}($errArgsStr);") + out.puts(s"throw new ${ksErrorName(err)}(${errArgsStr.mkString(", ")});") out.dec out.puts("}") } diff --git a/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala index c0e7b5e1d..60a19cf9a 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala @@ -1057,17 +1057,21 @@ class CppCompiler( } override def attrValidateExpr( - attrId: Identifier, - attrType: DataType, + attr: AttrLikeSpec, checkExpr: Ast.expr, err: KSError, - errArgs: List[Ast.expr] + useIo: Boolean, + expected: Option[Ast.expr] = None ): Unit = { - val errArgsStr = errArgs.map(translator.translate).mkString(", ") + val errArgsStr = expected.map(expression) ++ List( + expression(Ast.expr.InternalName(attr.id)), + if (useIo) expression(Ast.expr.InternalName(IoIdentifier)) else nullPtr, + expression(Ast.expr.Str(attr.path.mkString("/", "/", ""))) + ) importListSrc.addKaitai("kaitai/exceptions.h") outSrc.puts(s"if (!(${translator.translate(checkExpr)})) {") outSrc.inc - outSrc.puts(s"throw ${ksErrorName(err)}($errArgsStr);") + outSrc.puts(s"throw ${ksErrorName(err)}(${errArgsStr.mkString(", ")});") outSrc.dec outSrc.puts("}") } diff --git a/shared/src/main/scala/io/kaitai/struct/languages/GoCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/GoCompiler.scala index 267be931c..2c8b00cc2 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/GoCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/GoCompiler.scala @@ -531,16 +531,20 @@ class GoCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def ksErrorName(err: KSError): String = GoCompiler.ksErrorName(err) override def attrValidateExpr( - attrId: Identifier, - attrType: DataType, + attr: AttrLikeSpec, checkExpr: Ast.expr, err: KSError, - errArgs: List[Ast.expr] + useIo: Boolean, + expected: Option[Ast.expr] = None ): Unit = { - val errArgsStr = errArgs.map(translator.translate).mkString(", ") + val errArgsStr = expected.map(expression) ++ List( + expression(Ast.expr.InternalName(attr.id)), + if (useIo) expression(Ast.expr.InternalName(IoIdentifier)) else "nil", + expression(Ast.expr.Str(attr.path.mkString("/", "/", ""))) + ) out.puts(s"if !(${translator.translate(checkExpr)}) {") out.inc - val errInst = s"kaitai.New${err.name}($errArgsStr)" + val errInst = s"kaitai.New${err.name}(${errArgsStr.mkString(", ")})" val noValueAndErr = translator.returnRes match { case None => errInst case Some(r) => s"$r, $errInst" diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala index c30936146..503fa8b68 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala @@ -1132,16 +1132,20 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) } override def attrValidateExpr( - attrId: Identifier, - attrType: DataType, + attr: AttrLikeSpec, checkExpr: Ast.expr, err: KSError, - errArgs: List[Ast.expr] + useIo: Boolean, + expected: Option[Ast.expr] = None ): Unit = { - val errArgsStr = errArgs.map(translator.translate).mkString(", ") + val errArgsStr = expected.map(expression) ++ List( + expression(Ast.expr.InternalName(attr.id)), + if (useIo) expression(Ast.expr.InternalName(IoIdentifier)) else "null", + expression(Ast.expr.Str(attr.path.mkString("/", "/", ""))) + ) out.puts(s"if (!(${translator.translate(checkExpr)})) {") out.inc - out.puts(s"throw new ${ksErrorName(err)}($errArgsStr);") + out.puts(s"throw new ${ksErrorName(err)}(${errArgsStr.mkString(", ")});") out.dec out.puts("}") } diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JavaScriptCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JavaScriptCompiler.scala index ee29bb8ca..de0872fde 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/JavaScriptCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/JavaScriptCompiler.scala @@ -560,16 +560,20 @@ class JavaScriptCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def ksErrorName(err: KSError): String = JavaScriptCompiler.ksErrorName(err) override def attrValidateExpr( - attrId: Identifier, - attrType: DataType, + attr: AttrLikeSpec, checkExpr: Ast.expr, err: KSError, - errArgs: List[Ast.expr] + useIo: Boolean, + expected: Option[Ast.expr] = None ): Unit = { - val errArgsStr = errArgs.map(translator.translate).mkString(", ") + val errArgsStr = expected.map(expression) ++ List( + expression(Ast.expr.InternalName(attr.id)), + if (useIo) expression(Ast.expr.InternalName(IoIdentifier)) else "null", + expression(Ast.expr.Str(attr.path.mkString("/", "/", ""))) + ) out.puts(s"if (!(${translator.translate(checkExpr)})) {") out.inc - out.puts(s"throw new ${ksErrorName(err)}($errArgsStr);") + out.puts(s"throw new ${ksErrorName(err)}(${errArgsStr.mkString(", ")});") out.dec out.puts("}") } diff --git a/shared/src/main/scala/io/kaitai/struct/languages/LuaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/LuaCompiler.scala index 4c4f9c97c..6ccb47506 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/LuaCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/LuaCompiler.scala @@ -402,24 +402,21 @@ class LuaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def ksErrorName(err: KSError): String = LuaCompiler.ksErrorName(err) override def attrValidateExpr( - attrId: Identifier, - attrType: DataType, + attr: AttrLikeSpec, checkExpr: Ast.expr, err: KSError, - errArgs: List[Ast.expr] + useIo: Boolean, + expected: Option[Ast.expr] = None ): Unit = { - val errArgsCode = errArgs.map(translator.translate) + val actualStr = expression(Ast.expr.InternalName(attr.id)) out.puts(s"if not(${translator.translate(checkExpr)}) then") out.inc val msg = err match { case _: ValidationNotEqualError => { - val (expected, actual) = ( - errArgsCode.lift(0).getOrElse("[expected]"), - errArgsCode.lift(1).getOrElse("[actual]") - ) - s""""not equal, expected " .. $expected .. ", but got " .. $actual""" + val expectedStr = expected.get + s""""not equal, expected " .. $expectedStr .. ", but got " .. $actualStr""" } - case _ => "\"" + ksErrorName(err) + "\"" + case _ => expression(Ast.expr.Str(ksErrorName(err))) } out.puts(s"error($msg)") out.dec diff --git a/shared/src/main/scala/io/kaitai/struct/languages/PHPCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/PHPCompiler.scala index 42ed1fa16..5e182535a 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/PHPCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/PHPCompiler.scala @@ -480,16 +480,20 @@ class PHPCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def ksErrorName(err: KSError): String = PHPCompiler.ksErrorName(err) override def attrValidateExpr( - attrId: Identifier, - attrType: DataType, + attr: AttrLikeSpec, checkExpr: Ast.expr, err: KSError, - errArgs: List[Ast.expr] + useIo: Boolean, + expected: Option[Ast.expr] = None ): Unit = { - val errArgsStr = errArgs.map(translator.translate).mkString(", ") + val errArgsStr = expected.map(expression) ++ List( + expression(Ast.expr.InternalName(attr.id)), + if (useIo) expression(Ast.expr.InternalName(IoIdentifier)) else "null", + expression(Ast.expr.Str(attr.path.mkString("/", "/", ""))) + ) out.puts(s"if (!(${translator.translate(checkExpr)})) {") out.inc - out.puts(s"throw new ${ksErrorName(err)}($errArgsStr);") + out.puts(s"throw new ${ksErrorName(err)}(${errArgsStr.mkString(", ")});") out.dec out.puts("}") } diff --git a/shared/src/main/scala/io/kaitai/struct/languages/PythonCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/PythonCompiler.scala index 1d227dcf1..318fc1d8b 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/PythonCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/PythonCompiler.scala @@ -887,16 +887,20 @@ class PythonCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def ksErrorName(err: KSError): String = PythonCompiler.ksErrorName(err) override def attrValidateExpr( - attrId: Identifier, - attrType: DataType, + attr: AttrLikeSpec, checkExpr: Ast.expr, err: KSError, - errArgs: List[Ast.expr] + useIo: Boolean, + expected: Option[Ast.expr] = None ): Unit = { - val errArgsStr = errArgs.map(translator.translate).mkString(", ") + val errArgsStr = expected.map(expression) ++ List( + expression(Ast.expr.InternalName(attr.id)), + if (useIo) expression(Ast.expr.InternalName(IoIdentifier)) else "None", + expression(Ast.expr.Str(attr.path.mkString("/", "/", ""))) + ) out.puts(s"if not ${translator.translate(checkExpr)}:") out.inc - out.puts(s"raise ${ksErrorName(err)}($errArgsStr)") + out.puts(s"raise ${ksErrorName(err)}(${errArgsStr.mkString(", ")})") out.dec } diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RubyCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RubyCompiler.scala index c956f29c0..539bf7182 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/RubyCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/RubyCompiler.scala @@ -464,14 +464,18 @@ class RubyCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def ksErrorName(err: KSError): String = RubyCompiler.ksErrorName(err) override def attrValidateExpr( - attrId: Identifier, - attrType: DataType, + attr: AttrLikeSpec, checkExpr: Ast.expr, err: KSError, - errArgs: List[Ast.expr] + useIo: Boolean, + expected: Option[Ast.expr] = None ): Unit = { - val errArgsStr = errArgs.map(translator.translate).mkString(", ") - out.puts(s"raise ${ksErrorName(err)}.new($errArgsStr) if not ${translator.translate(checkExpr)}") + val errArgsStr = expected.map(expression) ++ List( + expression(Ast.expr.InternalName(attr.id)), + if (useIo) expression(Ast.expr.InternalName(IoIdentifier)) else "nil", + expression(Ast.expr.Str(attr.path.mkString("/", "/", ""))) + ) + out.puts(s"raise ${ksErrorName(err)}.new(${errArgsStr.mkString(", ")}) if not ${translator.translate(checkExpr)}") } def types2class(names: List[String]) = names.map(type2class).mkString("::") diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/CommonReads.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/CommonReads.scala index 899585e67..c95762e53 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/CommonReads.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/CommonReads.scala @@ -115,5 +115,5 @@ trait CommonReads extends LanguageCompiler { * @param attr attribute to run validations for */ def attrValidateAll(attr: AttrLikeSpec) = - attr.valid.foreach(valid => attrValidate(attr.id, attr, valid)) + attr.valid.foreach(valid => attrValidate(attr.id, attr, valid, true)) } diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/GenericChecks.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/GenericChecks.scala index b6321970e..449f874cc 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/GenericChecks.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/GenericChecks.scala @@ -42,7 +42,7 @@ trait GenericChecks extends LanguageCompiler with EveryReadIsExpression { attr.valid.foreach { (valid) => typeProvider._currentIteratorType = Some(attr.dataTypeComposite) if (bodyShouldDependOnIo.map(shouldDepend => validDependsOnIo(valid) == shouldDepend).getOrElse(true)) { - attrValidate(id, attr, valid) + attrValidate(id, attr, valid, false) } } diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/ValidateOps.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/ValidateOps.scala index 39290579c..3006a6e42 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/components/ValidateOps.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/components/ValidateOps.scala @@ -13,17 +13,17 @@ trait ValidateOps extends ExceptionNames { val translator: AbstractTranslator val typeProvider: ClassTypeProvider - def attrValidate(attrId: Identifier, attr: AttrLikeSpec, valid: ValidationSpec): Unit = { + def attrValidate(attrId: Identifier, attr: AttrLikeSpec, valid: ValidationSpec, useIo: Boolean): Unit = { valid match { case ValidationEq(expected) => - attrValidateExprCompare(attrId, attr, Ast.cmpop.Eq, expected, ValidationNotEqualError(attr.dataTypeComposite)) + attrValidateExprCompare(attrId, attr, Ast.cmpop.Eq, expected, ValidationNotEqualError(attr.dataTypeComposite), useIo) case ValidationMin(min) => - attrValidateExprCompare(attrId, attr, Ast.cmpop.GtE, min, ValidationLessThanError(attr.dataTypeComposite)) + attrValidateExprCompare(attrId, attr, Ast.cmpop.GtE, min, ValidationLessThanError(attr.dataTypeComposite), useIo) case ValidationMax(max) => - attrValidateExprCompare(attrId, attr, Ast.cmpop.LtE, max, ValidationGreaterThanError(attr.dataTypeComposite)) + attrValidateExprCompare(attrId, attr, Ast.cmpop.LtE, max, ValidationGreaterThanError(attr.dataTypeComposite), useIo) case ValidationRange(min, max) => - attrValidateExprCompare(attrId, attr, Ast.cmpop.GtE, min, ValidationLessThanError(attr.dataTypeComposite)) - attrValidateExprCompare(attrId, attr, Ast.cmpop.LtE, max, ValidationGreaterThanError(attr.dataTypeComposite)) + attrValidateExprCompare(attrId, attr, Ast.cmpop.GtE, min, ValidationLessThanError(attr.dataTypeComposite), useIo) + attrValidateExprCompare(attrId, attr, Ast.cmpop.LtE, max, ValidationGreaterThanError(attr.dataTypeComposite), useIo) case ValidationAnyOf(values) => val bigOrExpr = Ast.expr.BoolOp( Ast.boolop.Or, @@ -37,15 +37,10 @@ trait ValidateOps extends ExceptionNames { ) attrValidateExpr( - attrId, - attr.dataTypeComposite, + attr, checkExpr = bigOrExpr, err = ValidationNotAnyOfError(attr.dataTypeComposite), - errArgs = List( - Ast.expr.InternalName(attrId), - Ast.expr.InternalName(IoIdentifier), - Ast.expr.Str(attr.path.mkString("/", "/", "")) - ) + useIo ) case ValidationExpr(expr) => blockScopeHeader @@ -56,40 +51,37 @@ trait ValidateOps extends ExceptionNames { translator.translate(Ast.expr.InternalName(attrId)) ) attrValidateExpr( - attrId, - attr.dataTypeComposite, + attr, expr, ValidationExprError(attr.dataTypeComposite), - List( - Ast.expr.InternalName(attrId), - Ast.expr.InternalName(IoIdentifier), - Ast.expr.Str(attr.path.mkString("/", "/", "")) - ) + useIo ) blockScopeFooter } } - def attrValidateExprCompare(attrId: Identifier, attr: AttrLikeSpec, op: Ast.cmpop, expected: Ast.expr, err: KSError): Unit = { + def attrValidateExprCompare( + attrId: Identifier, + attr: AttrLikeSpec, + op: Ast.cmpop, + expected: Ast.expr, + err: KSError, + useIo: Boolean + ): Unit = { attrValidateExpr( - attrId, - attr.dataTypeComposite, + attr, checkExpr = Ast.expr.Compare( Ast.expr.InternalName(attrId), op, expected ), err = err, - errArgs = List( - expected, - Ast.expr.InternalName(attrId), - Ast.expr.InternalName(IoIdentifier), - Ast.expr.Str(attr.path.mkString("/", "/", "")) - ) + useIo = useIo, + expected = Some(expected) ) } - def attrValidateExpr(attrId: Identifier, attrType: DataType, checkExpr: Ast.expr, err: KSError, errArgs: List[Ast.expr]): Unit = {} + def attrValidateExpr(attr: AttrLikeSpec, checkExpr: Ast.expr, err: KSError, useIo: Boolean, expected: Option[Ast.expr] = None): Unit = {} def handleAssignmentTempVar(dataType: DataType, id: String, expr: String): Unit def blockScopeHeader: Unit def blockScopeFooter: Unit From 7d830fadd066b92e95bcd3bea5046646a09014d4 Mon Sep 17 00:00:00 2001 From: Petr Pucil Date: Fri, 13 Oct 2023 23:25:08 +0200 Subject: [PATCH 90/90] Fix Java 7 compat: add `final` in allocateIOFixed() --- .../main/scala/io/kaitai/struct/languages/JavaCompiler.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala index 503fa8b68..cb9dc4d5e 100644 --- a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala +++ b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala @@ -430,7 +430,7 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig) override def allocateIOFixed(varName: Identifier, size: String): String = { val ioName = idToStr(IoStorageIdentifier(varName)) - out.puts(s"$kstreamName $ioName = new ByteBufferKaitaiStream($size);") + out.puts(s"final $kstreamName $ioName = new ByteBufferKaitaiStream($size);") ioName }