From 2f0c0bf75d2852403f1e9fb38d3ad815ce9241d4 Mon Sep 17 00:00:00 2001 From: Lex Date: Wed, 29 Jul 2020 18:09:36 +0200 Subject: [PATCH 01/24] Rework user, dev and code doc --- docs/bof.rst | 3 - docs/index.rst | 18 +- docs/man/dev.rst | 130 +++++++ docs/{ => man}/images/bof_levels.png | Bin docs/{ => man}/images/bof_spec.png | Bin docs/{ => man}/images/knx_fields.png | Bin docs/man/images/knx_frame.png | Bin 0 -> 8738 bytes docs/man/knx.rst | 321 +++++++++++++++++ docs/man/user.rst | 206 +++++++++++ docs/usermanual.rst | 511 --------------------------- 10 files changed, 672 insertions(+), 517 deletions(-) create mode 100644 docs/man/dev.rst rename docs/{ => man}/images/bof_levels.png (100%) rename docs/{ => man}/images/bof_spec.png (100%) rename docs/{ => man}/images/knx_fields.png (100%) create mode 100644 docs/man/images/knx_frame.png create mode 100644 docs/man/knx.rst create mode 100644 docs/man/user.rst delete mode 100644 docs/usermanual.rst diff --git a/docs/bof.rst b/docs/bof.rst index fd58ba0..95cf57e 100644 --- a/docs/bof.rst +++ b/docs/bof.rst @@ -1,6 +1,3 @@ -Source code documentation -========================= - .. automodule:: bof :members: :undoc-members: diff --git a/docs/index.rst b/docs/index.rst index d4e8197..e7fcfd2 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -4,8 +4,20 @@ Boiboite Opener Framework's documentation ========================================= .. toctree:: - :maxdepth: 3 - :caption: Table of content: + :maxdepth: 2 + :caption: User manual + + man/user + man/knx + +.. toctree:: + :maxdepth: 2 + :caption: Developer manual + + man/dev + +.. toctree:: + :maxdepth: 2 + :caption: Source code documentation - usermanual bof diff --git a/docs/man/dev.rst b/docs/man/dev.rst new file mode 100644 index 0000000..47a143e --- /dev/null +++ b/docs/man/dev.rst @@ -0,0 +1,130 @@ +Notice +====== + +This section is intended for contributors, either for improving existing parts +(core, existing implementation) or adding new protocol implementations. Before +going further, please consider the following notice. + +Code quality requirements +------------------------- + +:Quality: + + We like clean code and expect contributions to be PEP-8 compliant as much as + possible (even though we don't test for it). New code should be readable easily + and maintainable. And remember: if you need to use "and" while explaining what + your function does, then you can probably split it. + +:Genericity: + + Part of the code (the "core") is used by all protocol implementations. When + you add code to the core, please make sure that it does not cause issues in + protocol-specific codes. Also, if you write or find out that code in + implementations can be made generic and added to the core, feel free to do + it. + +:Unit tests: + + We use Python's ``unittest`` to write unit tests. When working on BOF, please + write or update unit tests! They are in ``tests/``. You can run all unit tests + with: ``python -m unittest discover -s tests``. + +Comments and documentation +-------------------------- + +:Docstrings: + + Modules, functions, classes, methods start with docstrings written in + ReStructuredText. Docstrings are extracted to build the ReadTheDocs source + code documentation using Sphinx. We use a not-so-strict format, but you + should at least make sure that docstrings are useful to the reader, contain + the appropriate details and have a valid and consistent format. You can also + rely on the following model:: + + """Brief description of the module, function, class, method. + + A few details on how, where, when and why to use it. + + :param first: Description of param "first": type, usage, origin + Second line of description if one isn't enough. + :param second: Description of param "second" + :returns: The value that is returned, if any. + :raises BOFProgrammingError: if misused + + Usage example:: + + if there is any interest in adding such example, please do so. + """ + +Git branching +------------- + +We follow the "successful git branching model" described `here +`_. In a nutshell: + +* Branch from ``master`` for hotfixes +* Work on ``dev`` for small changes +* Create specific ``feature`` branches from ``dev`` for big changes +* Don't work on ``master`` + +Report issues +------------- + +Report bugs, ask questions or request for missing documentation and new features +by submitting an issue with GitHub. For bugs, please describe your problem as +clearly as you can. + +Architecture +============ + +The library has the following structure:: + + ../bof + ├── base.py + ├── byte.py + ├── frame.py + ├── __init__.py + ├── knx + │   ├── __init__.py + │   ├── knxdevice.py + │   ├── knxframe.py + │   ├── knxnet.json + │   └── knxnet.py + │── network.py + └── spec.py + +There is two distinct types of source files in BOF: The core content, used by +all implementations (in ``bof/``) and protocol-specific content, in subfolders +in ``bof/`` (ex: KNX implementation is in ``bof/knx/``). + +Core components are described below. Main principles of protocol implementations +are described in the next section. + +:Error and logging: + + ``base.py`` contains base exceptions classes for BOF and logging functions + (based on ``logging``). Developer-defined exceptions can be added. They + must inherit from base class ``BOFError``. + +:Converters: + + Module ``byte`` contain byte-conversion functions that are used directly in + protocol implementation classes. Any missing conversion class (from a type or + from a specific format, such as IPv4) should be added here and not directly + to implementations. + +:Network connection: + + Basic class for TCP and UDP asynchronous network connection is in + ``bof/network.py``. Such class should be used by protocol implementation's + connection classes (ex: ``KnxNet`` inherits ``UDP``). + +:Implementations base class: + + ``spec.py`` and ``frame.py`` contain base classes to use in implementation: + specification file parsing base class, frame, block and field base classes. + +(TODO) Extend BOF +================= + +TODO diff --git a/docs/images/bof_levels.png b/docs/man/images/bof_levels.png similarity index 100% rename from docs/images/bof_levels.png rename to docs/man/images/bof_levels.png diff --git a/docs/images/bof_spec.png b/docs/man/images/bof_spec.png similarity index 100% rename from docs/images/bof_spec.png rename to docs/man/images/bof_spec.png diff --git a/docs/images/knx_fields.png b/docs/man/images/knx_fields.png similarity index 100% rename from docs/images/knx_fields.png rename to docs/man/images/knx_fields.png diff --git a/docs/man/images/knx_frame.png b/docs/man/images/knx_frame.png new file mode 100644 index 0000000000000000000000000000000000000000..be111e83c28637e615be025153d3c10445fff86b GIT binary patch literal 8738 zcmbVRdpy(o|6k{lE=M}2(~VH4q${(Hp^&jLc41?)NwOKXvDxgxgiaS-+)k+!g>LRC zMM~jx<8(pgnrkN^_Y`xP@q2&N`CfnL_s6dX^Xc<`zh9T<>-K)VcFcw3pt(qU5ex>? z#5>wkV6bmg;Q82s`QY>B==e1lY`#E(^^gRG`f&p}FaxCBmnQ=RJU|#CF+kcIAP_#m z!A8DZAAhk=P>7L$BLT0#dXUhU>&NBze%XV7BjDTNsO<=IHxn}hq^$`Wd>~DY;HGHi zm;FBe9KlzD;58ty$-~5C^OrS_#K->&9ToylRoDhOB1tronYjfPPoSE9*(2kKL%2f0 zmu)6UBe)Te0un?L7R>qb)|VsWvN_OOBo?UhRc5e-`&G(NTSI3q}m zV|*bZ@Co`##03bt-Q)`qQ<|8?mI&oo7r2jOhzrVD8tO<82NKB$xPouahohNNISy&d zBiIFF{T)p$nAA`|ro9y7gl7tExdsRfohEY<`LWF?G%+CFLWJCZ8{&f_}zbN%58io{nWk@Ex1ME(R9JlzLHMcW2Dpe-C2H1ohfp@m5pFN73| zAu%lk_+T!MWy<4;3ARXrgDW~%8bl2vC}ape&e0ie;bcyfQ>p$=egS-mDbm6{RO}eQ zwGYBrgt;n!M&QR>?xP?eA?*YX3K_yay79K-q1(`G5XnY!0;zuM0F`b-5Qdfo<#@q}c6iPxQJ~X*0TVcXFk=>d0W`1}YUCiW|AVL)8z^yRmE)a&?5b)uP5(_E*C=Y_4 z49l>$^I&=ixjYeFK*BjX;GFCbbQB+fc5xu_2`*+~3~n&V741vJ5(FL$K1viqHRps- zT|La~DI~j4CkH%|>h2a`Vat@;kwf83dzu^Ep8yXQk+^~|4hJg>FvW2Z_UKSK!Oxiu zM8M&g{v-)X?qp)`FZbY4g^nm9l1mYCU7f)qRfuB=%`s92(H01b^LLR--Egi78rDzY zZ)--DGtGi{j;>?{+ku56DlpAjC!x5T z$#`@ZGiQ|0UxqhP_)0|1K{P2R)ZN6+R)Uo|(gToqH@-8*R~pC=g}N}QRGup#0FHEI z1CwA#uI?5{U*}L?wp|d8>?{``!@_U`IwMqsvGd^5m<$ln_HsVXgDoSOi7oi1CMJOr zCoDFYMVE-}5d=brZ-|{NERbnxM{s7iIg>1iW_DD%E7H}+l^7ag?=KL*F<7Cy8=FVK z(oKV%uy`qp6~t!onPQ6AMJ6&aqx%xw?fCX=Iu68O5Q8N*rGQL@;|2op1lT}?GmVXu zxdjMKv1AL^Fu5bz6wMKZ`q|Q?6rQg@EO8{VWIhBI2JeFLk^8w3s1yuBXf7Z)5QSk# zsw;^p!bxfVfEb-k^GBP={K9PEWV=v>JKbD@VG_tJ6Ojkg4=EvF;8L`Y%wLKKKu8D@ zfrnc#OCokKQJ^>;aGI@LL;^G%Bt8TuA%(4=`2(Izqg?ty}2% z!L|sGI_aeu5v3)6yD2fmC3^qbZAHzfrLU$go)na1PoBES-_kH5dPS?FQ45(Z#5l?G zLPINBMwi7Ue88ou~3^WE>AJbru;{&jC{X8m<<8{%2`MLj`v*SYdu6G;4}m}x}&$H}IoD~neeWx40l zEF-7ysGM>t$~IiveV1aT7`=1eYmafaP(hf?-}&{#mfP_Wcqb<(ia?ML?Tz&DxmY3} z&c`Qi^r|+xzA`pqZsWTI6V`yRqpU5$!2S9gx#b0IgzwR^*%{UI^W9~YO@?y-&YYc0 z(W_MswzP$}B&O-Ao;-O{(w)qAV=#JWnxDk19DD9PL&+qk<%A2MRfoVi#1CQA34JRegC{I+qRt=9WEG_ST0|&BICvlc$}UU)4%$B zw#?wGKA3&W(&E*-p8O^Gs*19e{}AL+(>R96o;{dg;}VeKGM@ICw5C4&)UQrMwbcIN z(#+p}TUt8P&aW>s+_sajBO&riV@EgQquR`-wA$H##k>nTb$`+aPa`&BvlA)& zH7bq8vl&@uHuLHcHC;sbdOW3E0s^f z5+OL`Pn&AwU0kBFg(s1&x7x|`DAkIn(0Hp88&Ov=)bZ(i3@o(rc7b)nuKxX*$4 zn%Zvt->+Q0ytlzqSbEcQay^H`k(T{H-WjI`N^0ti{nkC*i*I|=ONyAPSzlifS+Wo+2E-&XEK0y@jkq~SfXs^oBBNbE}`edl{dHi zw^~{?yeXna6Pi|-RHg(^S_hz=we763_Zvkr2oqN!&QTxYZSE){(I(6zwYwK>{#yR9P z>n+B{bOz%bi_IQ;veMg4RaKVx`ThI%S^UO)Xvx<0S8Y??dp#ApHTdA7MT>aq8LxY? zY;JAF6p&Z?={?>nVyvE7w(qXdp8Hwevr~h_P?FN^g0-~a=eFfh zN%4^&W@h~f%=>n;{W&*wCLygqth}Q>AR0*7x0BFMuWc$1{vbTlBd#dJt9jb0F?Cd_ zRowcoDzLt0tq&6;om2hZvpFEs@t+)DKJ_lfxbCt;f^!8h2jLt78?$1&*js?X90(uK z(%adjmXhcEMswB2;c}1jq)emTch>ekt1PpL9{KTn=fTLb!OA9df9TnUgo2u;Onr4; zpTDZfc>8vDCNp{dk~I^Lbj_m5Uf0$d;5+txj!QP>XB^bf%E`)dEN&G$U+Z~uM|64U zZjYY3o7;!}>*e`MM5=y51i7Q8X?9&~!rL|aqo0FpD#{p;g<>Nrtkoa<-ECSb%4&Pv zfD1o-_z( zdv?{>aEG^7iR4v#X}IxfRccvU!K@w6Qec$iyxsHm{rCNGI%t}!tLwG#7<6iclELxw zYnT*Az9=-5Pkm1>FNswHCfT5*4JB+6G~}Jb#jnYJ{hD$;-!msIExY+_B(O$hlm9?% zLL_bM*1mzp%m|f|`}Cy2$t$0GYI3Fh4c4nKwX+7i?>=9>u|0BM4wpPMx;Q^ls;eSp z)~b7u{?(f-EEvy|jK}**41jm4SE*tnre4+LM9wo+Iy*U`LO<@6_SNAb9Y7l5^p0t$ zn!F1GPF%WlsqS`=XVp1mWkhPSb$8+Y`@5GJlHBa=554Q`Jbm)yJH3sjUL>M^+EfT3 zT%740JodS($$0Z-S72{5Vf!}xvp@cDkw^+zEEY=Gz7zjx0(m+)xo_s)4FDv%O_E4T z-Pv0obZ1S5zERy|KO&d1YZ}%+9JJaJG(SAlF8a;wi z>YxJFX4W#c)4impj~_o?7@?AZ>e^eEP2FN-L>q1|EmDl}67)k#_Dzb%hT1%@UcD;5 zyL-{%#Z*7PjLPSey_K1YM}MIoua4H#De)d^y5VCt)>rR*`t<2%)zvhh0JsQ9lmiSv zdb6QnWoJ!Y!!^(P@~9|0g+z+i+xOA-`jod(ZNd@mKk(9a1PXq5Hnfx_mgw;@$* zQ2r+B(^JFcy1LiizxOpTG|UZZx$Wfaoapws^86AVGXjIb7#I zI+ac17oV?IZJq7c)C>*SKh&R+#|??R4stJUrqc27;Uk+Z&6wp8l0%pIBu+Cd zBxWh~TQVWDl2={z-xG@K)YH%W_(!nxZgEUWYYW!Z#C2-$^7{OV2O(F5FRNa>a05XJ zJjXO*N`gqft1NC47m9CvHf^B zt-Ja{Zc~%TlZpyj_`&?lwCrckoJ{!*gdfC4oPuEi7sJla!*VzBx+(1Ib*BWwP9C`(A9Gk~qnwgfCWw*DLROc3y zoAu~oD{=juvu(L{-uiBp)w*Lz?#|aI9xU5bS@k6)*ib8Gnv3|$moL8xE)4bkWB;Cj z*#jyUuZo#7YKxLVWlP;qyiBq8j2$>?I zI++z{Q#)L~cld9)cnhX%>N?a5_ia=n)iaB`K~2?X4WvBz3gtfw?Mzp9sa8jQP;rOf z7n(zVPZK{hBrc|@$DP=!bCI*(^Fh7V0Xw>OmGLSSh}Zue zPU+J}m!rAL18*wkK)U_6AVqqeEcL$yQZD>+naLhmT`a5-F%&>Y;_AFUeJ07a-4H1h-b^B&Eq ztd+iqtzWco;lg#&A0mUROdCHuJiHhjeejQ?T3fGXX7<-a&D?4;-Fpyp?EU1ZsYLJ$ z2bTT*HRwC*tlHzD=K6{?=r>R7TCH-=R_tAh>01%meiSrEms2VTYuMphtcZ6%b)sg* zJA;O8Uudz*<3XGHTQX<_3w|M&e=KC~@N8*6=#gfa8ipxuRm2Qh9i8+UX#A0#wvujW=1 zBVA*@&E9aWcjZ0<;lCD;7p!|c)cR^<$+f4EE#TzTZqRx`#SyTIiY^%bevZh!#IJ4% zNxYGmc14tsaH~{fqz0}m{T=GESs0KX#vWAaHvw@|BD^E=Du~)Z^a=k}sy%xLmSGl{ ztXo}ERsUPZl7do=0_$(}w9`RaZw4G^LBIPAPzZB$Gq?3hfihyO0Prqiq}|&S)KXjo zxRe7f@Tgig=(vym*lY1l(6(dLs3m-9|FSn1iz=o^&_Mf{l@(=4IYXNOKNQk}{XKm4^r3%1`jT1!74-LbEBNc)BdhjUdM6|h&X`Q6$IYw#tD#lDFf zL-#+E?(u)0pSY3u`t?=k!TeugkH8S=cX+8{?Dd)Lp1{{)K(|Iee(YGmz4_n#08N%2 z1-9HKT7@eM@68WG8Td`rbY`2;A_=Z9!WO=(B2uv=^*} z230>4`c-Lb>X^L%(mV(%Zp)~#R)OaQOOyVZhd91{Y^BVdS)AUt7#dp?7?-QshDYBX z)M|#j%X|9W%I2FK;9ZV;4@3AITD1b_9o+rr5&Z^sf|Xym)=wTjkPX*Gcb#z1d-dVf zW#F!PRa0aA(np$TVDj&1p>iV_EZQPmnU{ZW$P!ZB?#PmQP<9~IA400{>a0v?bT%&a zm1}wg)}&hXy}laLX1a9OD)24Us;nC~Tuz)g0kqQuQ?tUT*~okRwY%V8{06W4ZJk{q zum+&X5|&M70O$n(>jepsX@3lv z!@8JPza?&s#cP0d(~0#Cl1g29lh?PA@ddyw^n=aUfQvbFF0^KT-|@M39-!~gi<%Ze z^6Qudy3Ed~2lMCIR6v-JQGrb^pfE!xe);8>(XmGF7C>z7VA@%Wkc<`EJb+_)3{3W< zHJ;i`1*+IU!^q#s=Pjj9RVnD=d-J!+M&4n)r1kEANE!>s|JI=*UN@J%C+rC%WM&5Z5%oY+L{UH0itxQG>^QoDnH(uYa0C0=V*Zgw{0G56Z>E8^N0N7_6@)&HO z+`KvE@^0RwXlkQw0L*U7hxZt4*Z^5>KV+q}@ad7tqJUSwKvo(V8EGySrU6yV=g!49 z25V#%cSL$>*-YQ~>+QU~@BcW#EaIJpG}aN`cQquZg2?>=vg=yd8Y_hxxY9wal%us| zkY5-;Mo>Utu*8 z@$4C;JW|cGB7=4egTa7f%n-~=oC=my~MID$~vSw_GDsW5olX0%Jf0!M*%~u%G%tDNFt=Tuw32HIgU`OSI((E z`T|*#m6w-SQ5$Iqb@8Wu`z;-~I9*4&diCn#@$obQA-20JrN??Uk8$hgWy>yuNk!>o zcgo3}1a)le51P7`ZXlG#q@P_5zIZ_ZJ#yI2(%S{)K`mzELgmWs%qv-0n~H*3Xd&G% zL}}JwL~uGOiU02LvExUNy6oF`pF*LmQbn_YgDv$1StA2<(a(A%VAxCmsnre4|DZ{? zo)j3)$Z!55H@7lD8!Vlk5FL(=?wg+cJXBB~{3m9tzmeS4 z)g|kAk$&pLi9bgQbD+s}TQAstB`QHZ*iyXJ$cO}{;1rRlKrudatv;82Pk$vCZqPyF z4~9frlAd1$L+*+)aB~6Pb1rShuN+K=KvN10+QRF=O@#mECTHmLHpWeRkjkL%B?`g3 zV7xIVCdQq?$Oe|(vT-Ahn2dS_rQhDcU*-2Ft7a0(*4ZCE1b_~HD;TDLCLS6{sYE7$ zO>ofu-7ObBJGD7B0^Eq^!4@EFbiuSTB_(%aVuHt>?Mm{NrIkr~DG=|0rUJ#tla+1W z#@n{l-3{p}6n8mEhugi?# zXd^gVcL}=~nBG0SYDRzNqeh(Wj)wdc4XyjDRT@jyxPXzcWNMUq!6s@?TZxoPqn%NS z3jVFp{n@y41k7e(umdj=yl(;X)B}PB8#m^pr>8gTNP7O!9+q^Zhl6Yj)ZS`lM*H!{ z9~GT(=nd{s2L(VhkiZK;TJaR<^STFhBuFF@8j^hc7)(sFZi`UV>qsi&bTWYxN{3n* zAboKLbfBsMY`O_bGq9!Ukbz;l_CrY=2Ac;3{r_*VZ}!`(rw?Ne@A@VjP=MjFB>Q~K H-UI&wv3s!? literal 0 HcmV?d00001 diff --git a/docs/man/knx.rst b/docs/man/knx.rst new file mode 100644 index 0000000..eb196d6 --- /dev/null +++ b/docs/man/knx.rst @@ -0,0 +1,321 @@ +KNX +=== + +KNX is a field bus protocol, mainly used for building management systems. BOF +implements KNXnet/IP, which is part of the KNX specification to link field KNX +components to the IP network. + +.. code-block:: + + from bof import knx + +Discover KNX devices +-------------------- + +The function `search()` from `bof.knx` lists the IP addresses of KNX devices +responding on an IP network. + +>>> from bof import knx +>>> knx.search("192.168.1.0/24") +['192.168.1.10'] + +The function `discover()` gathers information about a KNX device at a defined IP +address (or on multiple KNX devices on an address range) and stores it to a +``KnxDevice`` object. + +>>> from bof import knx +>>> device = knx.discover("192.168.1.10") +>>> print(device) +KnxDevice: Name=boiboite, MAC=00:00:54:ff:ff:ff, IP=192.168.1.10:3671 KNX=15.15.255 + +Connect to a device +------------------- + +.. code-block:: python + + from bof import knx, BOFNetworkError + + knxnet = knx.KnxNet() + try: + knxnet.connect("192.168.1.1", 3671) + # Do stuff + except BOFNetworkError as bne: + print(str(bne)) + finally: + knxnet.disconnect() + +The class ``KnxNet`` is used to connect to a KNX device (server or object). It +creates a UDP connection to a KNX device. ``connect`` can take an additionnal +``init`` parameter. When ``True``, a special connection request frame is sent to +the remote KNX device to agree on terms for the connection and "initializes" the +KNX exchange. This is required for some exchanges (ex: configuration requests), +but most requests can be sent without such initialization. + +Send and receive frames +----------------------- + +.. code-block:: python + + frame = knx.KnxFrame(type="DESCRIPTION REQUEST") + print(frame) + knxnet.send(frame) + response = knxnet.receive() + print(response) + +When a connection is established, one may start sending KNX frames to the +device. Frames are sent and received as bytes arrays, but they are represented +as ``KnxFrame`` objects within BOF. In the example below, we create a frame with +type ``DESCRIPTION REQUEST`` (asking the device to describe itself). The format +of such frame is extracted from BOF's KNX specification JSON file (see next +section for details). The object is converted to a byte array when sent to the +device. The ``response`` received is a byte array parsed into a ``KnxFrame`` +object. + +There are several ways to gather information about a created or received frame: + +.. code-block:: python + + >>> bytes(frame) + b'\x06\x10\x02\x03\x00\x0e\x08\x01\x7f\x00\x00\x01\xbe\x6d' + + >>> print(frame) + KnxFrame object: + [HEADER] +
+ + + + [BODY] + KnxBlock: control endpoint + + + + + + >>> print(frame.sid) + DESCRIPTION REQUEST + + >>> print(frame.attributes) + ['header_length', 'protocol_version', 'service_identifier', 'total_length', + 'control_endpoint', 'structure_length', 'host_protocol_code', 'ip_address', + 'port'] + +The content of a frame is a set of blocks and fields. The ordered list of fields +object (even fields in blocks and blocks within blocks) can be accessed as +follows: + +.. code-block:: python + + >>> for field in frame: + ... print(field) + ... +
+ + + + + + + + +Finally, one can access specific part of a frame by its name (part of the +structure, block, field) and access its properties. + +.. code-block:: python + + >>> print(frame.header) + KnxBlock: header +
+ + + + + >>> print(frame.header.total_length) + + + >>> print(frame.header.total_length.name) + total length + + >>> print(frame.header.total_length.value) + b'\x00\x0e' + + >>> print(frame.header.total_length.size) + 2 + +Understanding KNX frames +------------------------ + +Conforming to the KNX Standard v2.1, a KNX frame has a header and body. The +header's structure never changes but the body's structure varies according to +the type of frame (message) given in the header's ``service identifier`` +field. In this manual, we call "block" a set of "fields" (smallest part of the +frame, usually a byte or a byte array). The header and body are blocks, but can +(will) also contain nested blocks. + +.. figure:: images/knx_frame.png + +Frame, block and field objects inherit from ``BOFFrame``, ``BOFBlock`` and +``BOFField`` global structures. A ``KnxFrame`` contains a header and a body as +blocks (``KnxBlock``). A block contains a set of raw fields (``KnxField``) +and/or nested ``KnxBlock`` objects with a special structure (ex: ``HPAI`` is a +type of block with fixed fields). Finally, a ``KnxField`` object has three main +attributes: a ``name``, a ``size`` (number of bytes) and a ``value`` (as a byte +array). + +For instance, the format of a ``DESCRIPTION REQUEST`` message extracted from the +specification has the following structure: + +.. figure:: images/knx_fields.png + +Some interaction (not all of them) require to send a ``CONNECT REQUEST`` frame + beforehand to agree on the type of connection, the channel to use, etc. In the + example above, we directly send a ``DESCRIPTION REQUEST``, which expects a + ``DESCRIPTION RESPONSE`` from the server. + +Create frames +------------- + +Within a script using BOF, a ``KnxFrame`` can be built either from scratch +(creating each block and field one by one), from a raw byte array that is parsed +(usually a received frame) or by specifying the type of frame in the +constructor. + +.. code-block:: python + + empty_frame = knx.KnxFrame() + existing_frame = knx.KnxFrame(type="DESCRIPTION REQUEST") + received_frame = knx.KnxFrame(bytes=data) + +From the specification +++++++++++++++++++++++ + +The KNX standard describes a set of message types with different +format. Specific predefined blocks and identifiers are also written to KNX +Specification's JSON file. It has not been fully implemented yet so there may be +missing content, please refer to `bof/knx/knxnet.json` to know what is currently +supported. Obviously, the specification file content can be changed or a frame +can be built without referring to the specification, we discuss it further in +the "Advanced usage" section (not available yet). + +.. code-block:: python + + frame = knx.KnxFrame(type="DESCRIPTION REQUEST") + +A ``KnxFrame`` object based on a frame with the ``DESCRIPTION REQUEST`` service +identifier (sid) will be built according to this portion of the ``knxnet.json`` +specification file. + +.. code-block:: json + + { + "frame": [ + {"name": "header", "type": "HEADER"}, + {"name": "body", "type": "depends:service identifier"} + ], + "blocks": { + "DESCRIPTION REQUEST": [ + {"name": "control endpoint", "type": "HPAI"} + ], + "HEADER": [ + {"name": "header length", "type": "field", "size": 1, "is_length": true}, + {"name": "protocol version", "type": "field", "size": 1, "default": "10"}, + {"name": "service identifier", "type": "field", "size": 2}, + {"name": "total length", "type": "field", "size": 2} + ], + "HPAI": [ + {"name": "structure length", "type": "field", "size": 1, "is_length": true}, + {"name": "host protocol code", "type": "field", "size": 1, "default": "01"}, + {"name": "ip address", "type": "field", "size": 4}, + {"name": "port", "type": "field", "size": 2} + ] + }, + "codes" : { + "service identifier": { + "0203": "DESCRIPTION REQUEST" + } + } + } + +It should then have the following pattern: + +.. figure:: images/bof_spec.png + +In predefined frames, fields are empty except for optional fields, fields with a +default value or fields that store a length, which is evaluated automatically. +Some frames can be sent as is to a remote server, such as ``DESCRIPTION +REQUEST`` frames, but some of them require to fill the empty fields. + +From a byte array ++++++++++++++++++ + +A KnxFrame object can be created by parsing a raw byte array. This is what +happens when receiving a frame from a remote server. + +.. code-block:: python + + data = b'\x06\x10\x02\x03\x00\x0e\x08\x01\x7f\x00\x00\x01\xbe\x6d' + frame_from_byte = knx.KnxFrame(bytes=data) + received_frame = knxnet.receive() # received_frame is a KnxFrame object + +The format of the frame must be understood by BOF to be efficient (i.e. the +service identifier shall be recognized and described in the JSON specification +file). + +From scratch +++++++++++++ + +A frame can be created without referring to a predefined format, by manually +adding blocks and fields to the frame. The section "Advanced usage" (not +available yet) contains details on how to do so. + +.. code-block:: + + frame = knx.KnxFrame() + frame.header.service_identifier.value = b"\x02\x03" + hpai = knx.KnxBlock(type="HPAI") + frame.body.append(hpai) + print(frame) + + +(TODO) Modify frames +-------------------- + +Say we want to create a ``CONNECT REQUEST`` frame. Using the two previous +sections, here is how to do it. + +We first need information associated to the current connection (source ip and +port, stored within a ``KnxNet`` object after the UDP connection is +established). + +.. code-block:: python + + ip, port = knxnet.source + +We have to create the predefined frame with the appropriate format (blocks and +fields), but some of them are empty (values set to 0). We then have to fill some +of them that are required to be understood by the server. + +.. code-block:: python + + connectreq = knx.KnxFrame(type="CONNECT REQUEST") + + connectreq.body.control_endpoint.ip_address.value = ip + connectreq.body.control_endpoint.port.value = port + connectreq.body.data_endpoint.ip_address.value = ip + connectreq.body.data_endpoint.port.value = port + +Finally, we need to specify the type of connection we want to establish with the +server. The latter is sent as an identifier in the field +``connection_type_code``. The list of existing identifiers for this field are +defined in the KNX standard and reported to the JSON specification +file. Therefore, we can either set the ID manually, or refer to the +specification file. The content of the specification file can be accessed by +instantiating the singleton class ``KnxSpec``. From this object, the sections in +the JSON file can be accessed as properties (details in "Advanced Usage" (not +available yet)). + +.. code-block:: + + knxspecs = knx.KnxSpec() + connection_type = knxspecs.connection_types["Device Management Connection"] + connectreq.body.connection_request_information.connection_type_code.value = connection_type diff --git a/docs/man/user.rst b/docs/man/user.rst new file mode 100644 index 0000000..3667159 --- /dev/null +++ b/docs/man/user.rst @@ -0,0 +1,206 @@ +Overview +======== + +BOF (Boiboite Opener Framework) is a testing framework for field protocols +implementations and devices. It is a Python 3.6+ library that provides means to +send, receive, create, parse and manipulate frames from supported protocols. + +The library currently supports **KNXnet/IP**, which is our focus, but it can be +extended to other types of BMS or industrial network protocols. + +There are three ways to use BOF: + +* Automated: Use of higher-level interaction functions to discover devices and + start basic exchanges, without requiring to know anything about the protocol. + +* Standard: Perform more advanced (legitimate) operations. This requires the end + user to know how the protocol works (how to establish connections, what kind + of messages to send). + +* Playful: Modify every single part of exchanged frames and misuse the protocol + instead of using it (we fuzz devices with it). The end user should have + started digging into the protocol's specifications. + +.. figure:: images/bof_levels.png + +**Please note that targeting BMS systems can have a severe impact on buildings and +people and that BOF must be used carefully.** + +TL;DR +===== + +Clone repository:: + + git clone https://github.com/Orange-Cyberdefense/bof.git + +BOF is a Python 3.6+ library that should be imported in scripts. It has no +installer yet so you need to refer to the `bof` subdirectory which contains the +library (inside the repository) in your project or to copy the folder to your +project's folder. Then, inside your code (or interactively): + +.. code-block:: python + + import bof + +Now you can start using BOF! + +The following code samples interact using the building management system +protocol KNXnet/IP (the framework supports only this one for now). + +Discover devices on a network +----------------------------- + +>>> from bof import knx +>>> knx.search("192.168.1.0/24") +['192.168.1.10'] + +>>> from bof import knx +>>> device = knx.discover("192.168.1.10") +>>> print(device) +KnxDevice: Name=boiboite, MAC=00:00:54:ff:ff:ff, IP=192.168.1.10:3671 KNX=15.15.255 + +Send and receive packets +------------------------ + +.. code-block:: python + + from bof import knx, BOFNetworkError + + knxnet = knx.KnxNet() + try: + knxnet.connect("192.168.1.1", 3671) + frame = knx.KnxFrame(type="DESCRIPTION REQUEST") + print(frame) + knxnet.send(frame) + response = knxnet.receive() + print(response) + except BOFNetworkError as bne: + print(str(bne)) + finally: + knxnet.disconnect() + +Craft your own packets! +----------------------- + +.. code-block:: python + + from bof import knx + + frame = knx.KnxFrame() + frame.header.service_identifier.value = b"\x02\x03" + hpai = knx.KnxBlock(type="HPAI") + frame.body.append(hpai) + print(frame) + +---------------------- + +Basic usage +=========== + +Library content +--------------- + +.. code-block:: python + + import bof + from bof import byte + from bof import knx + from bof import knx, BOFNetworkError + +Global module content can be imported directly from ``bof``. Protocol-specific +content is in specific submodules (ex: ``bof.knx``). + +Network connection +------------------ + +BOF provides core class for TCP and UDP network connections, however they should +not be used directly, but inherited in protocol implementation network +connection classes (ex: ``KnxNet`` inherits ``UDP``). A connection class carries +information about a network connection and method to manage connection and +exchanges, that can vary depending on the protocol. + +Here is an example on how to establish connection using the ``knx`` submodule +(``3671`` is the default port for KNXnet/IP). + +.. code-block:: python + + knxnet = knx.KnxNet() + try: + knxnet.connect("192.168.1.1", 3671) + knxnet.send(knx.KnxFrame(type="DESCRIPTION REQUEST")) + response = knxnet.receive() + except BOFNetworkError as bne: + print(str(bne)) + finally: + knxnet.disconnect() + +Frames in BOF +------------- + +Network frames are sent and received as byte arrays. They can be divided into a +set of blocks, which contain a set of fields of varying sizes. + +In BOF, frames, blocks and fields are represented as objects (classes). A frame +(``BOFFrame``) has a header and a body, both of them being blocks +(``BOFBlock``). A block contains a set of raw fields (``BOFField``) and/or +nested ``BOFBlock`` objects with a special structure. + +Implementations inherit from these objects to build their own +specification-defined frames. They are described in BOF in a JSON specification +file, containing the definition of message codes, block types and frame +structures. The JSON file can change from one protocol to another but we +recommend that protocol use a basis (details are in the developer's manual). + +The class ``BOFSpec``, inherited in implementations, is a singleton class to +parse and store specification JSON files. This class is used in +protocol implementations, mainly to build frames, but one can also refer to +it in scripts. + +Code sample using KnxSpec: + +>>> knx.KnxSpec().codes["service identifier"] +{'0000': 'EMPTY', '0201': 'SEARCH REQUEST', '0202': 'SEARCH RESPONSE', '0203': +'DESCRIPTION REQUEST', '0204': 'DESCRIPTION RESPONSE', '0205': 'CONNECT +REQUEST', '0206': 'CONNECT RESPONSE', '0207': 'CONNECTIONSTATE REQUEST', '0208': +'CONNECTIONSTATE RESPONSE', '0209': 'DISCONNECT REQUEST', '020A': 'DISCONNECT +RESPONSE', '0310': 'CONFIGURATION REQUEST', '0311': 'CONFIGURATION ACK'} + +Error handling and logging +-------------------------- + +BOF has custom exceptions inheriting from a global custom exception class +``BOFError`` (code in `bof/base.py`): + +:BOFLibraryError: Library, files and import-related exceptions. +:BOFNetworkError: Network-related exceptions (connection errors, etc.). +:BOFProgrammingError: Misuse of the framework. + +.. code-block:: python + + try: + knx.connect("invalid", 3671) + except bof.BOFNetworkError as bne: + print("Connection failure: ".format(str(bne))) + +Logging features can be enabled for the entire framework. Global events will be +stored to a file (default name is ``bof.log``). One can make direct call to +bof's logger to record custom events. + +.. code-block:: python + + bof.enable_logging() + bof.log("Cannot send data to {0}:{1}".format(address[0], address[1]), level="ERROR") + +Other useful stuff +------------------ + +The framework comes with some useful functions used within the library but that can +be used in scripts as well. Refer to source code documentation for details. + +:Byte conversion: `bof/byte.py` contains functions for byte resize and + conversion to/from int, string, ipv4, bit list. + +.. code-block:: python + + x = bof.byte.from_int(1234) + x = bof.byte.resize(x, 1) # Truncates diff --git a/docs/usermanual.rst b/docs/usermanual.rst deleted file mode 100644 index 010e85a..0000000 --- a/docs/usermanual.rst +++ /dev/null @@ -1,511 +0,0 @@ -User manual -=========== - -Overview --------- - -BOF (Boiboite Opener Framework) is a testing framework for field protocols -implementations and devices. It is a Python 3.6+ library that provides means to -send, receive, create, parse and manipulate frames from supported protocols. - -The library currently supports **KNXnet/IP**, which is our focus, but it can be -extended to other types of BMS or industrial network protocols. - -There are three ways to use BOF: - -* Automated: Use of higher-level interaction functions to discover devices and - start basic exchanges, without requiring to know anything about the protocol. - -* Standard: Perform more advanced (legitimate) operations. This requires the end - user to know how the protocol works (how to establish connections, what kind - of messages to send). - -* Playful: Modify every single part of exchanged frames and misuse the protocol - instead of using it (we fuzz devices with it). The end user should have - started digging into the protocol's specifications. - -.. figure:: images/bof_levels.png - -**Please note that targeting BMS systems can have a severe impact on buildings and -people and that BOF must be used carefully.** - -TL;DR ------ - -Clone repository:: - - git clone https://github.com/Orange-Cyberdefense/bof.git - -BOF is a Python 3.6+ library that should be imported in scripts. It has no -installer yet so you need to refer to the `bof` subdirectory which contains the -library (inside the repository) in your project or to copy the folder to your -project's folder. Then, inside your code (or interactively): - -.. code-block:: python - - import bof - -Now you can start using BOF! - -The following code samples interact using the building management system -protocol KNXnet/IP (the framework supports only this one for now). - -Discover devices on a network -+++++++++++++++++++++++++++++ - -The function `search()` from `bof.knx` lists the IP addresses of KNX devices -responding on an IP network. - ->>> from bof import knx ->>> knx.search("192.168.1.0/24") -['192.168.1.10'] - -The function `discover()` gathers information about a KNX device at a defined IP -address (or on multiple KNX devices on an address range) and stores it to a -``KnxDevice`` object. - ->>> from bof import knx ->>> device = knx.discover("192.168.1.10") ->>> print(device) -KnxDevice: Name=boiboite, MAC=00:00:54:ff:ff:ff, IP=192.168.1.10:3671 KNX=15.15.255 - -Send and receive packets -++++++++++++++++++++++++ - -.. code-block:: python - - from bof import knx, BOFNetworkError - - knxnet = knx.KnxNet() - try: - knxnet.connect("192.168.1.1", 3671) - frame = knx.KnxFrame(type="DESCRIPTION REQUEST") - print(frame) - knxnet.send(frame) - response = knxnet.receive() - print(response) - except BOFNetworkError as bne: - print(str(bne)) - finally: - knxnet.disconnect() - -Craft your own packets! -+++++++++++++++++++++++ - -.. code-block:: python - - from bof import knx - - frame = knx.KnxFrame() - frame.header.service_identifier.value = b"\x02\x03" - hpai = knx.KnxBlock(type="HPAI") - frame.body.append(hpai) - print(frame) - ----------------------- - -Basics ------- - -Structure and imports -+++++++++++++++++++++ - -.. code-block:: python - - import bof - from bof import byte - from bof import knx - from bof import knx, BOFNetworkError - -Global module content can be imported directly from ``bof``. Protocol-specific -content is in specific submodules (ex: ``bof.knx``). - -The library has the following structure:: - - ../bof - ├── base.py - ├── byte.py - ├── frame.py - ├── __init__.py - ├── knx - │   ├── __init__.py - │   ├── knxdevice.py - │   ├── knxframe.py - │   ├── knxnet.json - │   └── knxnet.py - └── network.py - -Error handling and logging -++++++++++++++++++++++++++ - -BOF has custom exceptions inheriting from a global custom exception class -``BOFError`` (code in `bof/base.py`): - -:BOFLibraryError: Library, files and import-related exceptions. -:BOFNetworkError: Network-related exceptions (connection errors, etc.). -:BOFProgrammingError: Misuse of the framework. - -.. code-block:: python - - try: - knx.connect("invalid", 3671) - except bof.BOFNetworkError as bne: - print("Connection failure: ".format(str(bne))) - -Logging features can be enabled for the entire framework. Global events will be -stored to a file (default name is ``bof.log``). One can make direct call to -bof's logger to record custom events. - -.. code-block:: python - - bof.enable_logging() - bof.log("Cannot send data to {0}:{1}".format(address[0], address[1]), level="ERROR") - -Other useful stuff -++++++++++++++++++ - -The framework comes with some useful functions used within the library but that can -be used in scripts as well. Refer to source code documentation for details. - -:Byte conversion: `bof/byte.py` contains functions for byte resize and - conversion to/from int, string, ipv4, bit list. - -.. code-block:: python - - x = bof.byte.from_int(1234) - x = bof.byte.resize(x, 1) # Truncates - -Network connection ------------------- - -So far, BOF only supports the KNXnet/IP protocol (using KNX field protocol). New -protocols should follow the same pattern. Below, ``3671`` is the default port -for KNXnet/IP. - -.. code-block:: python - - knxnet = knx.KnxNet() - try: - knxnet.connect("192.168.1.1", 3671) - knxnet.send(knx.KnxFrame(type="DESCRIPTION REQUEST")) - response = knxnet.receive() - except BOFNetworkError as bne: - print(str(bne)) - finally: - knxnet.disconnect() - -A ``KnxNet`` object carries information about a network connection and method to -manage connection and exchanges. - -connect/disconnect -++++++++++++++++++ - -.. code-block:: python - - connect(self, ip:str, port:int=3671, init:bool=False) -> object - disconnect(self, in_error:bool=False) -> object - -``connect`` takes an additionnal ``init`` parameter. When ``True``, a ``CONNECT -REQUEST`` frame is sent to the remote KNX device to initialize the connection -and a ``DISCONNECT REQUEST`` will be sent automatically when disconnecting. The -``KnxNet`` object stores data associated to the current connection. - -When a connection is established and initialized, the following attributes are -set: - -:channel: Communication channel ID set by the remote KNX device and given in - the ``CONNECT RESPONSE`` frame. - -send/receive -++++++++++++ - -.. code-block:: python - - send(self, data, address:tuple=None) -> int - receive(self, timeout:float=1.0) -> object - send_receive(self, data:bytes, address:tuple=None, timeout:float=1.0) -> - object - -``send`` and ``receive`` exchange data with the remote server as byte -arrays. One can directly send a raw byte array or a ``KnxFrame`` object which -will be converted to a byte array. Received frames are parsed into a -``KnxFrame`` object (returned by ``receive``) but can still be accessed as raw -bytes. See next section for more information on ``KnxFrame`` objects. - -``send_receive`` is just a merge of send and receive, meaning that -``send_receive()`` is equivalent to ``send() ; receive()``. This is useful for -protocols such as KNX that do TCP stuff over UDP. - -Here is how to send and receive frames as ``KnxFrame`` objects (``send`` can -also take a raw byte array instead of a ``KnxFrame`` object). How to build such -object is described in the next section. - -.. code-block:: python - - request = knx.KnxFrame(type="DESCRIPTION REQUEST") - knxnet.send(request) - response = knxnet.receive() - print(response) # Response is a KnxFrame object - -BOF frames ----------- - -Frames are sent and received as byte arrays. They can be divided into a set of -blocks, which contain a set of fields of varying sizes. - -In BOF, frames, blocks and fields are represented as objects (classes). A frame -(``BOFFrame``) has a header and a body, both of them being blocks -(``BOFBlock``). A block contains a set of raw fields (``BOFField``) and/or -nested ``BOFBlock`` objects with a special structure. - -Implementations (so far, KNX) inherit from these objects to build their own -specification-defined frames. They are described in BOF in a JSON specification -file, containing the definition of message codes, block types and frame -structures. The class ``BOFSpec``, inherited in implementations, is a singleton -class to parse and store specification JSON files. See "Developer manual" for -more information (not available yet). - -KNX frames ----------- - -Conforming to the KNX Standard v2.1, the header's structure never changes and -the body's structure varies according to the type of the frame given in the -header's ``service identifier`` field. For instance, the format of a -``DESCRIPTION REQUEST`` message extracted from the specification has the -following content. - -.. figure:: images/knx_fields.png - -Frame, block and field objects inherit from ``BOFFrame``, ``BOFBlock`` and -``BOFField`` global structures. A frame (``KnxFrame``) has a header and a body, -both of them being blocks (``KnxBlock``). A block contains a set of raw fields -(``KnxField``) and/or nested ``KnxBlock`` objects with a special structure (ex: -``HPAI`` is a type of block with fixed fields). Finally, a ``KnxField`` object -has three main attributes: a ``name``, a ``size`` (number of bytes) and a -``value`` (as a byte array). - -Create frames -+++++++++++++ - -Within a script using BOF, a ``KnxFrame`` can be built either from scratch -(creating each block and field one by one), from a raw byte array that is parsed -(usually a received frame) or by specifying the type of the frame in the -constructor. - -.. code-block:: python - - empty_frame = knx.KnxFrame() - existing_frame = knx.KnxFrame(type="DESCRIPTION REQUEST") - received_frame = knx.KnxFrame(frame=data, source=address) - -From the specification -"""""""""""""""""""""" - -The KNX standard describes a set of message types with different -format. Specific predefined blocks and identifiers are also written to KNX -Specification's JSON file. It has not been fully implemented yet so there may be -missing content, please refer to `bof/knx/knxnet.json` to know what is currently -supported. Obviously, the specification file content can be changed or a frame -can be built without referring to the specification, we discuss it further in -the "Advanced usage" section (not available yet). - -.. code-block:: python - - frame = knx.KnxFrame(type="DESCRIPTION REQUEST") - -A ``KnxFrame`` object based on a frame with the ``DESCRIPTION REQUEST`` service -identifier (sid) will be built according to this portion of the ``knxnet.json`` -specification file. - -.. code-block:: json - - { - "service identifiers": { - "DESCRIPTION REQUEST": {"id": "0203"} - }, - "bodies": { - "DESCRIPTION REQUEST": [ - {"name": "control endpoint", "type": "HPAI"} - ] - }, - "blocktypes": { - "HEADER": [ - {"name": "header length", "type": "field", "size": 1, "is_length": true}, - {"name": "protocol version", "type": "field", "size": 1, "default": "10"}, - {"name": "service identifier", "type": "field", "size": 2}, - {"name": "total length", "type": "field", "size": 2} - ], - "HPAI": [ - {"name": "structure length", "type": "field", "size": 1, "is_length": true}, - {"name": "host protocol code", "type": "field", "size": 1, "default": "01"}, - {"name": "ip address", "type": "field", "size": 4}, - {"name": "port", "type": "field", "size": 2} - ] - } - } - -It should then have the following pattern: - -.. figure:: images/bof_spec.png - -In predefined frames, fields are empty except for optional fields, fields with a -default value or fields that store a length, which is evaluated automatically. -Some frames can be sent as is to a remote server, such as ``DESCRIPTION -REQUEST`` frames, but some of them require to fill the empty fields (see `Modify -frames`_ below). - -From a byte array -""""""""""""""""" - -A KnxFrame object can be created by parsing a raw byte array. This is what -happens when receiving a frame from a remote server. - -.. code-block:: python - - data = b'\x06\x10\x02\x03\x00\x0e\x08\x01\x7f\x00\x00\x01\xbe\x6d' - frame_from_byte = knx.KnxFrame(bytes=data) - received_frame = knxnet.receive() # received_frame is a KnxFrame object - -The format of the frame must be understood by BOF to be efficient (i.e. the -service identifier shall be recognized and described in the JSON specification -file). - -From scratch -"""""""""""" - -A frame can be created without referring to a predefined format, by manually -adding blocks and fields to the frame. The section "Advanced usage" (not -available yet) contains details on how to do so. - -.. code-block:: - - frame = knx.KnxFrame() - frame.header.service_identifier.value = b"\x02\x03" - hpai = knx.KnxBlock(type="HPAI") - frame.body.append(hpai) - print(frame) - -Read frames -+++++++++++ - -There are several ways to gather information about a frame: - -.. code-block:: python - - >>> bytes(frame) - b'\x06\x10\x02\x03\x00\x0e\x08\x01\x7f\x00\x00\x01\xbe\x6d' - - >>> print(frame) - KnxFrame object: - [HEADER] -
- - - - [BODY] - KnxBlock: control endpoint - - - - - - >>> print(frame.sid) - DESCRIPTION REQUEST - - >>> print(frame.attributes) - ['header_length', 'protocol_version', 'service_identifier', 'total_length', - 'control_endpoint', 'structure_length', 'host_protocol_code', 'ip_address', - 'port'] - -The content of a frame is a set of blocks and fields. The ordered list of fields -object (even fields in blocks and blocks within blocks) can be accessed as -follows: - -.. code-block:: python - - >>> for field in frame: - ... print(field) - ... -
- - - - - - - - - >>> print(frame.fields) - [, - , , , - , , , - ] - -Finally, one can access specific part of a frame by name (part of the structure, -block, field) and access its properties. - -.. code-block:: python - - >>> print(frame.header) - KnxBlock: header -
- - - - - >>> print(frame.header.total_length) - - - >>> print(frame.header.total_length.name) - total length - - >>> print(frame.header.total_length.value) - b'\x00\x0e' - - >>> print(frame.header.total_length.size) - 2 - -Modify frames -+++++++++++++ - -Say we want to create a ``CONNECT REQUEST`` frame. Using the two previous -sections, here is how to do it. - -We first need information associated to the current connection (source ip and -port, stored within a ``KnxNet`` object after the UDP connection is -established). - -.. code-block:: python - - ip, port = knxnet.source - -We have to create the predefined frame with the appropriate format (blocks and -fields), but some of them are empty (values set to 0). We then have to fill some -of them that are required to be understood by the server. - -.. code-block:: python - - connectreq = knx.KnxFrame(type="CONNECT REQUEST") - - connectreq.body.control_endpoint.ip_address.value = ip - connectreq.body.control_endpoint.port.value = port - connectreq.body.data_endpoint.ip_address.value = ip - connectreq.body.data_endpoint.port.value = port - -Finally, we need to specify the type of connection we want to establish with the -server. The latter is sent as an identifier in the field -``connection_type_code``. The list of existing identifiers for this field are -defined in the KNX standard and reported to the JSON specification -file. Therefore, we can either set the ID manually, or refer to the -specification file. The content of the specification file can be accessed by -instantiating the singleton class ``KnxSpec``. From this object, the sections in -the JSON file can be accessed as properties (details in "Advanced Usage" (not -available yet)). - -.. code-block:: - - knxspecs = knx.KnxSpec() - connection_type = knxspecs.connection_types["Device Management Connection"] - connectreq.body.connection_request_information.connection_type_code.value = connection_type From 77869387edf2f142587bafe3656d35ff6779bf60 Mon Sep 17 00:00:00 2001 From: JulienBedel Date: Fri, 31 Jul 2020 10:26:03 +0200 Subject: [PATCH 02/24] chore(opcua): Add module for OPC UA implementation Add init file for OPC UA module, currently containing a simple docstring only. --- bof/opcua/__init__.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 bof/opcua/__init__.py diff --git a/bof/opcua/__init__.py b/bof/opcua/__init__.py new file mode 100644 index 0000000..1c24ca4 --- /dev/null +++ b/bof/opcua/__init__.py @@ -0,0 +1,13 @@ +""" +OPC UA protocol implementation for industrial automation. + +The implementation schemes and frame descriptions are based on **IEC 62541** +standard, **v1.04**. The specification is described in :file:`opcua.json`. + +A user should be able to create or alter any frame to both valid and invalid +format. Therefore, a user can copy, modify or replace the specification file to +include or change any content they want. + +The ``opcua`` submodule has the following content: +""" + From 76a01d1afd42df8ba46f8bf4126c5173335efb1d Mon Sep 17 00:00:00 2001 From: JulienBedel Date: Fri, 31 Jul 2020 11:45:47 +0200 Subject: [PATCH 03/24] feat(opcua): Add BOFSpec implementation to opcua - Add OpcuaSpec class in opcua module to implement BOFSpec. - Identified bug on custom spec file opening, not fixed yet. - Wrote unit tests for OpcuaSpec class init and methods. - Due to the bug affecting file opening, most of the tests won't pass. --- bof/opcua/__init__.py | 8 ++++ bof/opcua/opcua.json | 30 +++++++++++++ bof/opcua/opcuaframe.py | 92 +++++++++++++++++++++++++++++++++++++++ tests/jsons/invalid.json | 1 + tests/jsons/valid.json | 30 +++++++++++++ tests/test_opcua_frame.py | 70 +++++++++++++++++++++++++++++ 6 files changed, 231 insertions(+) create mode 100644 bof/opcua/opcua.json create mode 100644 bof/opcua/opcuaframe.py create mode 100644 tests/jsons/invalid.json create mode 100644 tests/jsons/valid.json create mode 100644 tests/test_opcua_frame.py diff --git a/bof/opcua/__init__.py b/bof/opcua/__init__.py index 1c24ca4..394c8aa 100644 --- a/bof/opcua/__init__.py +++ b/bof/opcua/__init__.py @@ -9,5 +9,13 @@ include or change any content they want. The ``opcua`` submodule has the following content: + +:opcuaframe: + Object representations of OPC UA frames and frame content (blocks and + fields) and specification details, with methods to build, alter or read + a frame or part of it. Available from direct import of the ``opcua`` + submodule (``from bof import opcua``). """ +from .opcuaframe import * + diff --git a/bof/opcua/opcua.json b/bof/opcua/opcua.json new file mode 100644 index 0000000..e624c9e --- /dev/null +++ b/bof/opcua/opcua.json @@ -0,0 +1,30 @@ +{ + "frame": [ + {"name": "header", "type": "HEADER"}, + {"name": "body", "type": "depends:message_type"} + ], + "blocks": { + "HEADER": [ + {"name": "message_type", "type": "field", "size": 3}, + {"name": "is_final", "type": "field", "size": 1}, + {"name": "message_size", "type": "field", "size": 4, "is_length": true} + ], + "HEL_BODY": [ + {"name": "protocol_version", "type": "field", "size": 4}, + {"name": "receive_buffer_size", "type": "field", "size": 4}, + {"name": "send_buffer_size", "type": "field", "size": 4}, + {"name": "max_message_size", "type": "field", "size": 4}, + {"name": "max_chunk_count", "type": "field", "size": 4}, + {"name": "endpoint_url", "type": "STRING", "size": 18} + ], + "STRING": [ + {"name": "string_length", "type": "field", "size": 4, "is_length": true}, + {"name": "string_value", "type": "field", "size": 14} + ] + }, + "codes" : { + "message_type": { + "HEL": "HEL_BODY" + } + } +} diff --git a/bof/opcua/opcuaframe.py b/bof/opcua/opcuaframe.py new file mode 100644 index 0000000..be42c00 --- /dev/null +++ b/bof/opcua/opcuaframe.py @@ -0,0 +1,92 @@ +""" +OPC UA frame handling +------------------ + +OPC UA frames handling implementation, implementing ``bof.frame``'s +``BOFSpec``, ``BOFFrame``, ``BOFBlock`` and ``BOFField`` classes. +See `bof/frame.py`. + +BOF should not contain code that is bound to a specific version of a protocol's +specifications. Therefore OPC UA frame structure and its content is described in +:file:`opcua.json`. +""" + +from os import path + +from ..base import BOFProgrammingError, to_property, log +from ..frame import BOFFrame, BOFBlock, BOFField +from ..spec import BOFSpec + +############################################################################### +# OPCUA SPECIFICATION CONTENT # +############################################################################### + +OPCUASPECFILE = "opcua.json" + +class OpcuaSpec(BOFSpec): + """Singleton class for OPC UA specification content usage. + Inherits ``BOFSpec``, see `bof/frame.py`. + + The default specification is ``opcua.json`` however the end user is free + to modify this file (add categories, contents and attributes) or create a + new file following this format. + + Usage example:: + + spec = OpcuaSpec() + block_template = spec.get_block_template(block_name="HEL_BODY") + item_template = spec.get_item_template("HEL_BODY", "protocol version") + message_structure = spec.get_association("message_type", "HEL") + """ + + def __init__(self): + """Initialize the specification object with a JSON file. + If `filepath` is not specified, we use a default one specified in + `OPCUASPECFILE`. + """ + #TODO: add support for custom file path (depends on BOFSpec) + filepath = path.join(path.dirname(path.realpath(__file__)), OPCUASPECFILE) + super().__init__(filepath) + + #-------------------------------------------------------------------------# + # Public # + #-------------------------------------------------------------------------# + + def get_item_template(self, block_name:str, item_name:str) -> dict: + """Returns an item template (dict of values) from a `block_name` and + a `field_name`. + + Note that an item template can represent either a field or a block, + depending on the "type" key of the item. + + :param block_name: Name of the block containing the item. + :param item_name: Name of the item to look for in the block. + """ + if block_name in self.blocks: + for item in self.blocks[block_name]: + if item['name'] == item_name: + return item + return None + + def get_block_template(self, block_name:str) -> list: + """Returns a block template (list of item templates) from a block name. + + :param block_name: Name of the block we want the template from. + """ + return self._get_dict_value(self.blocks, block_name) if block_name else None + + def get_association(self, code_name:str, identifier) -> str: + """Returns the value associated to an `identifier` inside a `code_name` + association table. See `opcua.json` + usage example to better + understand the association table concept. + + :param identifier: Key we want the value from. + :code name: Association table name we want to look into for identifier + match. + """ + #TODO: add support for bytes codes names (if needed in the specs ?) + if code_name in self.codes: + for association in self.codes[code_name]: + if identifier == association: + return self.codes[code_name][association] + return None diff --git a/tests/jsons/invalid.json b/tests/jsons/invalid.json new file mode 100644 index 0000000..b9669c0 --- /dev/null +++ b/tests/jsons/invalid.json @@ -0,0 +1 @@ +{{} \ No newline at end of file diff --git a/tests/jsons/valid.json b/tests/jsons/valid.json new file mode 100644 index 0000000..e624c9e --- /dev/null +++ b/tests/jsons/valid.json @@ -0,0 +1,30 @@ +{ + "frame": [ + {"name": "header", "type": "HEADER"}, + {"name": "body", "type": "depends:message_type"} + ], + "blocks": { + "HEADER": [ + {"name": "message_type", "type": "field", "size": 3}, + {"name": "is_final", "type": "field", "size": 1}, + {"name": "message_size", "type": "field", "size": 4, "is_length": true} + ], + "HEL_BODY": [ + {"name": "protocol_version", "type": "field", "size": 4}, + {"name": "receive_buffer_size", "type": "field", "size": 4}, + {"name": "send_buffer_size", "type": "field", "size": 4}, + {"name": "max_message_size", "type": "field", "size": 4}, + {"name": "max_chunk_count", "type": "field", "size": 4}, + {"name": "endpoint_url", "type": "STRING", "size": 18} + ], + "STRING": [ + {"name": "string_length", "type": "field", "size": 4, "is_length": true}, + {"name": "string_value", "type": "field", "size": 14} + ] + }, + "codes" : { + "message_type": { + "HEL": "HEL_BODY" + } + } +} diff --git a/tests/test_opcua_frame.py b/tests/test_opcua_frame.py new file mode 100644 index 0000000..869adf9 --- /dev/null +++ b/tests/test_opcua_frame.py @@ -0,0 +1,70 @@ +"""unittest for ``bof.opcua``. + +- OPC UA specification gathering +""" + +import unittest +from bof import opcua, BOFLibraryError + +class Test01OpcuaSpec(unittest.TestCase): + """Test class for specification class building from JSON file.""" + def test_01_opcua_spec_instantiate_default(self): + """Test that the current `opcua.json` default file is valid.""" + try: + spec = opcua.OpcuaSpec() + except BOFLibraryError: + self.fail("Default opcua.json should not raise BOFLibraryError.") + def test_02_opcua_spec_instantiate_custom_valid_json(self): + """Test that a custom and valid opcua spec file works as expected.""" + try: + spec = opcua.OpcuaSpec("./jsons/valid_opcua.json") + except BOFLibraryError: + self.fail("Valid json spec should not raise BOFLibraryError.") + def test_03_opcua_spec_instantiate_custom_invalid_json(self): + """Test that a custom and invalid opcua spec raises exception.""" + with self.assertRaises(BOFLibraryError): + spec = opcua.OpcuaSpec("./jsons/invalid.json") + def test_04_opcua_spec_instantiate_custom_invalid_path(self): + """Test an invalid custom path raises exception.""" + with self.assertRaises(BOFLibraryError): + spec = opcua.OpcuaSpec("./jsons/unexisting.json") + def test_05_opcua_spec_property_access_valid(): + """Test that a JSON element can be accessed as property""" + spec = opcua.OpcuaSpec("./jsons/valid_opcua.json") + frame_template = spec.frame + self.assertEqual(frame_template[0]["name"], "header") + def test_05_opcua_spec_property_access_invalid(): + """Test that a unexisting JSON element can't be accessed as property""" + spec = opcua.OpcuaSpec("./jsons/valid_opcua.json") + with self.assertRaises(AttributeError): + frame_template = spec.unexisting + def test_06_opcua_spec_get_blocks_valid(): + """Test that we can get block from spec as expected""" + spec = opcua.OpcuaSpec("./jsons/valid_opcua.json") + block_template = spec.get_block_template(block_name="HEADER") + self.assertEqual(block_template[0]["name"], "message_type") + def test_07_opcua_spec_get_blocks_invalid(): + """Test that an invalid block request returns None""" + spec = opcua.OpcuaSpec("./jsons/valid_opcua.json") + block_template = spec.get_block_template(block_name="INVALID") + self.assertEqual(block_template, None) + def test_08_opcua_spec_get_item_valid(): + """Test that we can get an item from spec as expected""" + spec = opcua.OpcuaSpec("./jsons/valid_opcua.json") + item_template = spec.get_template("HEL_BODY", "protocol_version") + self.assertEqual(item_template["name"], "protocol_version") + def test_09_opcua_spec_get_item_invalid(): + """Test that an invalid item request returns None""" + spec = opcua.OpcuaSpec("./jsons/valid_opcua.json") + item_template = spec.get_template("INVALID", "INVALID") + self.assertEqual(item_template, None) + def test_10_get_association_valid(): + """Test that a valid association is returned as expeted""" + spec = opcua.OpcuaSpec("./jsons/valid_opcua.json") + message_structure = spec.get_association("message_type", "HEL") + self.assertEqual(message_structure, "HEL_BODY") + def test_11_get_association_invalid(): + """Test that an ivalid association returns None""" + spec = opcua.OpcuaSpec("./jsons/valid_opcua.json") + message_structure = spec.get_association("INVALID", "INVALID") + self.assertEqual(message_structure, None) From 851ceaa9984db96c90bdd9c767f80a63de878c68 Mon Sep 17 00:00:00 2001 From: JulienBedel Date: Fri, 31 Jul 2020 13:48:23 +0200 Subject: [PATCH 04/24] feat(opcua): Add BOFField implementation to opcua - Add OpcuaField class in opcua module to implement BOFField. - Wrote unit tests for OpcuaField class init and needed methods from BOFField. - Some tests might be generic to BOFField and could be moved. --- bof/opcua/opcuaframe.py | 23 +++++++++++++++++++++++ tests/test_opcua_frame.py | 36 +++++++++++++++++++++++++++++++++++- 2 files changed, 58 insertions(+), 1 deletion(-) diff --git a/bof/opcua/opcuaframe.py b/bof/opcua/opcuaframe.py index be42c00..c3676d4 100644 --- a/bof/opcua/opcuaframe.py +++ b/bof/opcua/opcuaframe.py @@ -90,3 +90,26 @@ def get_association(self, code_name:str, identifier) -> str: if identifier == association: return self.codes[code_name][association] return None + +############################################################################### +# OPC UA FRAME CONTENT # +############################################################################### + +#-----------------------------------------------------------------------------# +# OPC UA fields (byte or byte array) representation # +#-----------------------------------------------------------------------------# + +class OpcuaField(BOFField): + """An ``OpcuaField`` is a set of raw bytes with a name, a size and a + content (``value``). Inherits ``BOFField``. See `frame.py`. + + Usage example:: + + # creating a field from raw parameters + field = opcua.OpcuaField(name="protocol version", value=b"1", size=4) + field.value = field.value = b'\x00\x00\x00\x02' + + # creating a field from a template + item_template_field = spec.get_item_template("HEL_BODY", "protocol version") + field = opcua.OpcuaField(**item_template_field) + """ \ No newline at end of file diff --git a/tests/test_opcua_frame.py b/tests/test_opcua_frame.py index 869adf9..6536aa7 100644 --- a/tests/test_opcua_frame.py +++ b/tests/test_opcua_frame.py @@ -7,7 +7,9 @@ from bof import opcua, BOFLibraryError class Test01OpcuaSpec(unittest.TestCase): - """Test class for specification class building from JSON file.""" + """Test class for specification class building from JSON file. + TODO: Some tests are generic to BOFSpec and could be moved. + """ def test_01_opcua_spec_instantiate_default(self): """Test that the current `opcua.json` default file is valid.""" try: @@ -68,3 +70,35 @@ def test_11_get_association_invalid(): spec = opcua.OpcuaSpec("./jsons/valid_opcua.json") message_structure = spec.get_association("INVALID", "INVALID") self.assertEqual(message_structure, None) + +class Test02OpcuaField(unittest.TestCase): + """Test class for field crafting and access. + Note that we don't test for the whole BOFField behavior, but + uniquely what we are using in OpcuaField. + TODO: Some tests are generic to BOFField and could be moved. + """ + def test_01_opcua_create_field_manual(self): + """Test that we can craft a field by hand and content is set""" + field = opcua.OpcuaField(name="protocol_version") + self.assertEqual(field.name, "protocol_version") + def test_02_opcua_create_field_template(self): + """Test that we can craft a field from a template and content is set""" + spec = opcua.OpcuaSpec() + item_template_field = spec.get_item_template("HEL_BODY", "protocol_version") + field = opcua.OpcuaField(**item_template_field) + self.assertEqual(field.name, "protocol_version") + def test_03_opcua_field_set(self): + """Test that a field value can get set as expected.""" + field = opcua.OpcuaField(name="protocol_version", size=4) + field.value = b'\x00\x00\x00\x01' + self.assertEqual(field.value, b'\x00\x00\x00\x01') + def test_04_opcua_field_set_large(self): + """Test that if we set a value that is to large, it will be cropped.""" + field = opcua.OpcuaField(name="protocol_version", size=4) + field.value = b'\x00\x00\x00\x00\x01' + self.assertEqual(field.value, b'\x00\x00\x00\x01') + def test_05_opcua_field_set_small(self): + """Test that if we set a value that is to small, it will be extended.""" + field = opcua.OpcuaField(name="protocol_version", size=4) + field.value = b'\x00\x00\x01' + self.assertEqual(field.value, b'\x00\x00\x00\x01') From 2893480934bc547694c5c979a1b97c39310df33e Mon Sep 17 00:00:00 2001 From: JulienBedel Date: Fri, 31 Jul 2020 16:01:12 +0200 Subject: [PATCH 05/24] feat(opcua): Add basic BOFBlock implementation to opcua Add OpcuaBlock class in opcua module to implement BOFBlock. The following functionalities are implemented : - creation of block from type name or template - support for nested blocks - support for inner block dependencies The following functionalities are NOT implemented yet : - creation of block from raw bytes value (next push goal) Units test have also been written for implemented functionalities. They are not perfect/exhaustive (but at least allowed to identify some bugs when coding). --- bof/opcua/opcua.json | 4 +- bof/opcua/opcuaframe.py | 152 +++++++++++++++++++++++++++++++++++++- tests/jsons/valid.json | 4 +- tests/test_opcua_frame.py | 60 ++++++++++++++- 4 files changed, 212 insertions(+), 8 deletions(-) diff --git a/bof/opcua/opcua.json b/bof/opcua/opcua.json index e624c9e..7afe3d4 100644 --- a/bof/opcua/opcua.json +++ b/bof/opcua/opcua.json @@ -7,7 +7,7 @@ "HEADER": [ {"name": "message_type", "type": "field", "size": 3}, {"name": "is_final", "type": "field", "size": 1}, - {"name": "message_size", "type": "field", "size": 4, "is_length": true} + {"name": "message_size", "type": "field", "size": 4} ], "HEL_BODY": [ {"name": "protocol_version", "type": "field", "size": 4}, @@ -18,7 +18,7 @@ {"name": "endpoint_url", "type": "STRING", "size": 18} ], "STRING": [ - {"name": "string_length", "type": "field", "size": 4, "is_length": true}, + {"name": "string_length", "type": "field", "size": 4}, {"name": "string_value", "type": "field", "size": 14} ] }, diff --git a/bof/opcua/opcuaframe.py b/bof/opcua/opcuaframe.py index c3676d4..ddfdba1 100644 --- a/bof/opcua/opcuaframe.py +++ b/bof/opcua/opcuaframe.py @@ -33,7 +33,7 @@ class OpcuaSpec(BOFSpec): Usage example:: - spec = OpcuaSpec() + spec = opcua.OpcuaSpec() block_template = spec.get_block_template(block_name="HEL_BODY") item_template = spec.get_item_template("HEL_BODY", "protocol version") message_structure = spec.get_association("message_type", "HEL") @@ -75,7 +75,7 @@ def get_block_template(self, block_name:str) -> list: """ return self._get_dict_value(self.blocks, block_name) if block_name else None - def get_association(self, code_name:str, identifier) -> str: + def get_association(self, code_name:str, identifier:str) -> str: """Returns the value associated to an `identifier` inside a `code_name` association table. See `opcua.json` + usage example to better understand the association table concept. @@ -112,4 +112,150 @@ class OpcuaField(BOFField): # creating a field from a template item_template_field = spec.get_item_template("HEL_BODY", "protocol version") field = opcua.OpcuaField(**item_template_field) - """ \ No newline at end of file + """ + + # For now there is no need to redefine getter and setter property from + # BOFField (base class attributes are compatible with OPC UA field spec) + +#-----------------------------------------------------------------------------# +# OPC UA blocks (set of fields) representation # +#-----------------------------------------------------------------------------# + +class OpcuaBlock(BOFBlock): + """Object representation of an OPC UA block. Inherits ``BOFBlock``. + See `frame.py`. + + An OpcuaBlock (as well as a BOFBlock) contains a set of items. + Those items can be fields as well as blocks, therefore creating + "nested blocks" or "sub-block". + + It is usually built from a template which gives its structure. + Values can also be specified to "fill" the block stucture. + If no structure is specified the block remains empty. + + Some block field's value (typically a sub-block type) may depend on the + value of another field. In that case the keyword "depends:" is used to + associate the variable to its value, based on given parameters to another + field. The process can be looked in details `__init__` method, and + understood from examples. + + Usage example:: + + # block creation directly from type + block = opcua.OpcuaBlock(name="parent-block", type="HEADER") + + # block creation from an item template + item_template_block = {"name": "header", "type": "HEADER"} + block = opcua.OpcuaBlock(item_template_block=item_template_block) + + # block creation from an item template with dependency (defaults is needed) + # see opcua.json to understand defaults parameter value. + item_template_block = {"name": "body", "type": "depends:message_type"} + block = opcua.OpcuaBlock( + item_template_block={"name": "body", "type": "depends:message_type"}, + defaults={"message_type": "HEL"}) + + # block customization "by hand" + block = opcua.OpcuaBlock(type="HEADER") + sub_block = opcua.OpcuaBlock(type="STRING") + block.append(sub_block) + + # we can access list block `fields` using + block.attributes + + # and access on of those `fields` with + block.protocol_version + + #TODO: add example with block from value once implemented + """ + + @classmethod + def factory(cls, item_template:dict, **kwargs) -> object: + """Returns either an `OpcuaBlock` or an `OpcuaField` depending on the + template specified item type. That's why it's a factory as a class method. + + :param item_template: item template representing sub-block or field. + + Keyword arguments: + + :param defaults: defaults values in a dict, needed to construct + blocks with dependencies, see example above. + :param value: bytes value to fill the block with TODO: implement + + """ + # case where item template represents a field (non-recursive) + if "type" in item_template and item_template["type"] == "field": + value = b'' + if "defaults" in kwargs and item_template["name"] in kwargs["defaults"]: + value = kwargs["defaults"][item_template["name"]] + return OpcuaField(**item_template, value=value) + # case where item template represents a sub-block (nested/recursive block) + else: + return OpcuaBlock(item_template_block=item_template, **kwargs) + + def __init__(self, **kwargs): + """Initialize the ``OpcuaBlock``. + + Keyword arguments: + + :param type: a string specifying block type (as found in json + specifications) to construct the block on. + :param item_template_block: item template dict corresponding to a + block (which is described in item_block_template['type']), + giving its structure to the OpcuaBlock. If a block type + is already specified this parameter won't be taken into + account. + :param defaults: defaults values in a dict, needed to construct + blocks with dependencies. + :param value: bytes value to fill the block with TODO: implement + + """ + self._spec = OpcuaSpec() + super().__init__(**kwargs) + + # we gather args values and set some default values first + defaults = kwargs["defaults"] if "defaults" in kwargs else {} + value = kwargs["value"] if "value" in kwargs else {} + + block_type = None + block_template = None + + # there are several way to get a block type, either by its name + # or by specifying a template + if "type" in kwargs: + block_type = kwargs["type"] + elif "item_template_block" in kwargs and "type" in kwargs["item_template_block"]: + item_template_block = kwargs["item_template_block"] + block_type = item_template_block["type"] + else: + log("No type or item_template_block specified, creating empty block") + return + + # if a dependency is found in block type + # looks for needed information in default arg + if block_type.startswith("depends:"): + dependency = to_property(block_type.split(":")[1]) + if dependency in defaults: + block_type = self._spec.get_association(dependency, defaults[dependency]) + else: + raise BOFProgrammingError("Dependecy '{0}' missing in defaults parameter".format(dependency)) + if not block_type: + raise BOFProgrammingError("Association not found for dependency '{0}'".format(dependency)) + log("Creating OpcuaBlock of type '{0}' from dependency '{1}'.".format(block_type, dependency)) + else: + log("Creating OpcuaBlock of type '{0}'.".format(block_type)) + + # for the moment, we set the block name to its type + self._name = block_type + + # if block template has not been we extract the block template according to the type found in item template + block_template = self._spec.get_block_template(block_type) + + if block_template: + for item_template in block_template: + new_item = self.factory(item_template, defaults=defaults, parent=self) + self.append(new_item) + else: + raise BOFProgrammingError("Block type '{0}' not found in specifications.".format(block_type)) + + return diff --git a/tests/jsons/valid.json b/tests/jsons/valid.json index e624c9e..7afe3d4 100644 --- a/tests/jsons/valid.json +++ b/tests/jsons/valid.json @@ -7,7 +7,7 @@ "HEADER": [ {"name": "message_type", "type": "field", "size": 3}, {"name": "is_final", "type": "field", "size": 1}, - {"name": "message_size", "type": "field", "size": 4, "is_length": true} + {"name": "message_size", "type": "field", "size": 4} ], "HEL_BODY": [ {"name": "protocol_version", "type": "field", "size": 4}, @@ -18,7 +18,7 @@ {"name": "endpoint_url", "type": "STRING", "size": 18} ], "STRING": [ - {"name": "string_length", "type": "field", "size": 4, "is_length": true}, + {"name": "string_length", "type": "field", "size": 4}, {"name": "string_value", "type": "field", "size": 14} ] }, diff --git a/tests/test_opcua_frame.py b/tests/test_opcua_frame.py index 6536aa7..ea94f77 100644 --- a/tests/test_opcua_frame.py +++ b/tests/test_opcua_frame.py @@ -4,7 +4,7 @@ """ import unittest -from bof import opcua, BOFLibraryError +from bof import opcua, BOFLibraryError, BOFProgrammingError class Test01OpcuaSpec(unittest.TestCase): """Test class for specification class building from JSON file. @@ -102,3 +102,61 @@ def test_05_opcua_field_set_small(self): field = opcua.OpcuaField(name="protocol_version", size=4) field.value = b'\x00\x00\x01' self.assertEqual(field.value, b'\x00\x00\x00\x01') + +class Test03OpcuaBlock(unittest.TestCase): + """Test class for block crafting and access.""" + def test_01_opcua_create_block_empty(self): + """Tests that an empty block can be created and returns an empty list of fields""" + block = opcua.OpcuaBlock() + self.assertEqual(block.content, []) + def test_02_opcua_create_block_template(self): + """Tests that a block can be created as expected from a template""" + item_template_block = {"name": "header", "type": "HEADER"} + block = opcua.OpcuaBlock(item_template_block=item_template_block) + self.assertEqual(block.content[0].name, "message_type") + def test_03_opcua_create_block_template_invalid(self): + """Tests that block creation with invalid template fail case is handled""" + with self.assertRaises(BOFProgrammingError): + item_template_block = {"name": "header", "type": "unknown"} + block = opcua.OpcuaBlock(item_template_block=item_template_block) + def test_04_opcua_create_block_type(self): + """Tests that a block can be created as expected from a type name""" + block = opcua.OpcuaBlock(type="HEADER") + self.assertEqual(block.content[0].name, "message_type") + def test_05_opcua_create_block_type_invalid(self): + """Tests that block creation with invalid type name fail case is handled""" + with self.assertRaises(BOFProgrammingError): + block = opcua.OpcuaBlock(type="unknown") + def test_06_opcua_create_mixed(self): + """Tests that in case of mixed block type specification + (type + template) only the type is kept""" + item_template_block = {"name": "header", "type": "unknown"} + block = opcua.OpcuaBlock(item_template_block=item_template_block, type="STRING") + self.assertEqual(block.content[0].name, "string_length") + self.assertNotEqual(block.content[0].name, "message_type") + def test_07_opcua_create_nested_block(self): + """Test for manual creation of nested block""" + block = opcua.OpcuaBlock(type="HEADER") + sub_block = opcua.OpcuaBlock(type="STRING") + block.append(sub_block) + self.assertEqual(block.string.attributes, ['string_length', 'string_value']) + def test_08_opcua_create_dependency_block(self): + """Test dependecy block creation""" + item_template_block = {"name": "body", "type": "depends:message_type"} + block = opcua.OpcuaBlock( + item_template_block={"name": "body", "type": "depends:message_type"}, + defaults={"message_type": "HEL"}) + self.assertEqual(block.name, "HEL_BODY") + def test_09_opcua_create_dependency_block_missing(self): + """Test nested block creation with missing defaults values""" + with self.assertRaises(BOFProgrammingError): + item_template_block = {"name": "body", "type": "depends:message_type"} + block = opcua.OpcuaBlock( + item_template_block={"name": "body", "type": "depends:message_type"}) + def test_10_opcua_create_nested_dependecy_block_wrong(self): + """Test nested block creation with wrong defaults values""" + with self.assertRaises(BOFProgrammingError): + item_template_block = {"name": "body", "type": "depends:message_type"} + block = opcua.OpcuaBlock( + item_template_block={"name": "body", "type": "depends:message_type"}, + defaults={"message_type": "unknown"}) From 3f44a9e89660d04ad85ad0121bb0990bb9b386d2 Mon Sep 17 00:00:00 2001 From: JulienBedel Date: Fri, 31 Jul 2020 16:29:30 +0200 Subject: [PATCH 06/24] feat(opcua): Add support for OpcuaBlock bytes fill Add functionality to fill block with a specified byte string at creation. --- bof/opcua/opcuaframe.py | 17 +++++++++-------- tests/test_opcua_frame.py | 8 ++++++++ 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/bof/opcua/opcuaframe.py b/bof/opcua/opcuaframe.py index ddfdba1..c1a8b02 100644 --- a/bof/opcua/opcuaframe.py +++ b/bof/opcua/opcuaframe.py @@ -137,7 +137,7 @@ class OpcuaBlock(BOFBlock): value of another field. In that case the keyword "depends:" is used to associate the variable to its value, based on given parameters to another field. The process can be looked in details `__init__` method, and - understood from examples. + understood from examples. They are not perfect/exhaustive (but at least allowed to identify some bugs when coding). Usage example:: @@ -155,6 +155,9 @@ class OpcuaBlock(BOFBlock): item_template_block={"name": "body", "type": "depends:message_type"}, defaults={"message_type": "HEL"}) + # fills block with value at creation (a type is still mandatory) + block = opcua.OpcuaBlock(type="STRING", value=14*b"\x01") + # block customization "by hand" block = opcua.OpcuaBlock(type="HEADER") sub_block = opcua.OpcuaBlock(type="STRING") @@ -165,8 +168,6 @@ class OpcuaBlock(BOFBlock): # and access on of those `fields` with block.protocol_version - - #TODO: add example with block from value once implemented """ @classmethod @@ -180,14 +181,14 @@ def factory(cls, item_template:dict, **kwargs) -> object: :param defaults: defaults values in a dict, needed to construct blocks with dependencies, see example above. - :param value: bytes value to fill the block with TODO: implement + :param value: bytes value to fill the block with """ # case where item template represents a field (non-recursive) if "type" in item_template and item_template["type"] == "field": value = b'' - if "defaults" in kwargs and item_template["name"] in kwargs["defaults"]: - value = kwargs["defaults"][item_template["name"]] + if "value" in kwargs and kwargs["value"]: + value = kwargs["value"][:item_template["size"]] return OpcuaField(**item_template, value=value) # case where item template represents a sub-block (nested/recursive block) else: @@ -207,7 +208,7 @@ def __init__(self, **kwargs): account. :param defaults: defaults values in a dict, needed to construct blocks with dependencies. - :param value: bytes value to fill the block with TODO: implement + :param value: bytes value to fill the block with """ self._spec = OpcuaSpec() @@ -253,7 +254,7 @@ def __init__(self, **kwargs): if block_template: for item_template in block_template: - new_item = self.factory(item_template, defaults=defaults, parent=self) + new_item = self.factory(item_template, defaults=defaults, value=value, parent=self) self.append(new_item) else: raise BOFProgrammingError("Block type '{0}' not found in specifications.".format(block_type)) diff --git a/tests/test_opcua_frame.py b/tests/test_opcua_frame.py index ea94f77..ba7e828 100644 --- a/tests/test_opcua_frame.py +++ b/tests/test_opcua_frame.py @@ -160,3 +160,11 @@ def test_10_opcua_create_nested_dependecy_block_wrong(self): block = opcua.OpcuaBlock( item_template_block={"name": "body", "type": "depends:message_type"}, defaults={"message_type": "unknown"}) + def test_11_opcua_create_block_with_value(self): + """Test block creation with a value to fill it""" + block = opcua.OpcuaBlock(type="STRING", value=14*b"\x01") + self.assertEqual(block.string_length.value, b'\x01\x01\x01\x01') + def test_12_opcua_create_block_with_value_missing_type(self): + """Test that a block value with not type returns an empty list as expected""" + block = opcua.OpcuaBlock() + self.assertEqual(block.content, []) \ No newline at end of file From 1d48172444d7363eaad1abb56f7ad2f0accec8b1 Mon Sep 17 00:00:00 2001 From: JulienBedel Date: Fri, 31 Jul 2020 16:59:09 +0200 Subject: [PATCH 07/24] fix(opcua): Add byte-by-byte filling (splitting) Add byte splitting when filling a block, otherwise the block will try to fill again and again from the top. --- bof/opcua/opcuaframe.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/bof/opcua/opcuaframe.py b/bof/opcua/opcuaframe.py index c1a8b02..d0bd897 100644 --- a/bof/opcua/opcuaframe.py +++ b/bof/opcua/opcuaframe.py @@ -256,6 +256,12 @@ def __init__(self, **kwargs): for item_template in block_template: new_item = self.factory(item_template, defaults=defaults, value=value, parent=self) self.append(new_item) + + # Cut value to fill byte by byte + if value: + if len(new_item) >= len(value): + break + value = value[len(new_item):] else: raise BOFProgrammingError("Block type '{0}' not found in specifications.".format(block_type)) From 9393f1f083cccd1be5dd2e4b5c1d9bd50d6f6b0c Mon Sep 17 00:00:00 2001 From: Lex Date: Fri, 31 Jul 2020 17:59:46 +0200 Subject: [PATCH 08/24] Developer documentation up to date for version 0.2.X --- bof/knx/knxframe.py | 1 - docs/man/dev.rst | 230 +++++++++++++++++++++++++++++++++++++++++++- docs/man/knx.rst | 96 ++++++++++++------ 3 files changed, 291 insertions(+), 36 deletions(-) diff --git a/bof/knx/knxframe.py b/bof/knx/knxframe.py index 66e625a..264640f 100644 --- a/bof/knx/knxframe.py +++ b/bof/knx/knxframe.py @@ -233,7 +233,6 @@ class KnxFrame(BOFFrame): "connection": "connection type code" } - # TODO def __init__(self, **kwargs): """Initialize a KnxFrame object from various origins using values from keyword argument (kwargs). diff --git a/docs/man/dev.rst b/docs/man/dev.rst index 47a143e..ad931ff 100644 --- a/docs/man/dev.rst +++ b/docs/man/dev.rst @@ -124,7 +124,229 @@ are described in the next section. ``spec.py`` and ``frame.py`` contain base classes to use in implementation: specification file parsing base class, frame, block and field base classes. -(TODO) Extend BOF -================= - -TODO +Extend BOF +========== + +BOF can (and should) be extended to other network protocols. If you feel like +contributing, here is how to start a new protocol implementation. + +Source files tree +----------------- + +The folder ``bof`` contains the library core functions. Subfolders, such as +``bof\knx`` contain implementations. Please create a new subfolder for your +implementation. + +You should need 3 main components for a protocol implementation, described +below: + +* A JSON file describing the protocol specification +* A connection class +* Frame, block and field building and parsing classes + +Create the specification +------------------------ + +BOF parses a JSON file that explains how the library should create and parse +frames in a defined protocol. The main objective of using an external file is +not to bind the code too tightly to the specification (to change both of them +more easily). This JSON file is used within the code in a specification class +inheriting from ``BOFSpec``. It is recommended to build your own spec class and +to not use ``BOFSpec`` directly. + +Write the JSON file ++++++++++++++++++++ + +The format of the JSON is "almost" up to you. We define 3 main categories that +BOF core can recognize, but you can add more or change them, as long as you +adapt the code in you subclasses. If you want not to rely on a JSON spec file, +you can, but you may loose all the benefits of using BOF :( + +The JSON file should be in your protocol's subdirectory and we recommend that +you use the following base. + +.. code-block:: json + + { + "frame": [ + {"name": "header", "type": "HEADER"}, + {"name": "body", "type": "depends:message type"} + ], + "blocks": { + "EMPTY": [ + {} + ], + "HEADER": [ + {"name": "header length", "type": "field", "size": 1, "is_length": true}, + {"name": "message type", "type": "field", "size": 1, "default": "01"}, + {"name": "total length", "type": "field", "size": 2} + ], + "HELLO": [ + {"name": "target otter", "type": "OTTER_DESC"} + ], + "OTTER_DESC": [ + {"name": "otter name", "type": "field", "size": 30}, + {"name": "age", "type": "field", "size": 1} + ], + }, + "codes" : { + "message type": { + "01": "HELLO" + } + } + } + +There are three categories in a specification JSON file: + +:frame: The fixed definition of the frame format. For instance, many protocols + have a frame with a fixed header and a varying body. +:blocks: The list of blocks (fixed set of fields and/or nested blocks. Blocks + can be complete frame body (ex: in the base JSON file, ``message type`` + is used to choose the body) or part of another block. +:codes: Tables to match received codes as bytes arrays with block types (blocks) + +.. warning:: + + You are free to use them or not. However, if you do not follow this format + you will have to create a class inheriting from ``BOFSpec`` in your protocol + implementation and either add methods and code in your subclass if you add + categories to the JSON file or overload methods from ``BOFSpec`` to change or + remove the handling of these three default categories. + +To sum up: + +* ``frame`` is the structure of the corresponding ``BOFFrame`` subclass + of your implementation. each entry is added to the list of blocks contained + in the frame. +* An entry is the definition of either a block with a specific type, referred by + its name in the ``blocks`` category, or a ``field``. A block can contain as many + nested blocks as required. +* The smallest item of a frame is a field, ``BOF`` will read blocks until it + find fields. A field have a few mandatory parameters, and some optional ones. + +:name: Mandatory name of the field +:size: Mandatory size of the field, in bytes +:type: ``field`` :) +:is_length: Optional boolean. If true, the value of this field is the size of + the block, and is updated when the block size changed +:default: Default value, if no value has been specified before (by the user or + by parsing an existing frame). + +Specification file parsing +++++++++++++++++++++++++++ + +Here is how the example JSON file above is used in the code: + +The protocol implementation shall refer to ``BOFSpec`` or a subclass of +``BOFSpec`` that parses your JSON file.:: + + class OtterSpec(BOFSpec): + """Otter specification class, using the content of otter's JSON file.""" + def __init__(self, filepath:str=None): + if not filepath: + filepath = path.join(path.dirname(path.realpath(__file__)), "otter.json") + super().__init__(filepath) + +By default, your implementation's frame class inheriting from ``BOFFrame`` will +read the ``frame`` category. Here, the frame will have two main parts: a header +and a body.:: + + {"name": "header", "type": "HEADER"}, + {"name": "body", "type": "depends:message type"} + +We notice that ``header`` has type ``HEADER`` which is a type of block, defined +in the ``blocks`` category. The block ``header`` will then filled according th +the type defined and contain three fields.:: + + "HEADER": [ + {"name": "header length", "type": "field", "size": 1, "is_length": true}, + {"name": "message type", "type": "field", "size": 1, "default": "01"}, + {"name": "total length", "type": "field", "size": 2} + ] + +A field has a set of attributes, discussed previously. When the frame is created +from the specification, blocks and fields are created but not filled, unless +there is a default value given (in command line or with the keyword ``default`` +in the JSON file). When created from parsing a byte array, the fields are filled +directly with received bytes. The final header should look like this.:: + + BOFBLock: header + BOFField: header_length: b'\x04' (1 byte) + BOFField: message_type: b'\x01' (1 byte) + BOFField: total_length: b'\x00\x23' (2 byte) + +The field ``total length`` is the complete size of the frame. You will have to +write some code in the frame to handle it, as well as any special field. Here is +an example from the ``KNX`` implementation:: + + if "total_length" in self._blocks["header"].attributes: + total = sum([len(block) for block in self._blocks.values()]) + self._blocks["header"].total_length._update_value(byte.from_int(total)) + +Now let's move to the body. Here, it contains only one block, but its content +changes entirely depending on the type of message: ``"type": "depends:message +type"``. This means that the parser will require the value of a field with name +``message type`` set previously (in the header, here). We'll need the category +``codes`` to match values with associated block types. ``codes`` is a dictionary +and each key is the name of a block. When extracting the value of ``message +type``, we'll search in ``codes["message type"]`` to know if there is a matching +block name for a value. If a value is ``\x01``, then the body block should be a +block ``HELLO``.:: + + "codes" : { + "message type": { + "01": "HELLO" + }, + } + +The block type ``HELLO`` contains a block ``OTTER_DESC``, so we build it as well +as a nested block. The final body should look like this:: + + BOFBLock: body + BOFBlock: target otter + BOFField: otter_name: b'seraph\x00\x00\x00 [...]' (30 bytes) + BOFField: age: b'\x02' (1 byte) + +Write connection classes +------------------------ + +Inheriting (or not) from ``UDP``, ``TCP`` or whatever from ``network.py``, it +(or they) should contain the protocol-specific connection steps. For instance, +if the protocol requires to send an init message, it should be implemented +here. You may or may not rely on methods from parent classes +(``connect/disconnect``, ``send/receive``). + +For instance, the class ``KnxNet`` (connection class for KNX) implements ``UDP`` +and overloads the ``receive()`` method to convert received bytes to a +``KnxFrame`` object. + +.. code-block:: python + + def receive(self, timeout:float=1.0) -> object: + data, address = super().receive(timeout) + return KnxFrame(bytes=data, source=address) + +We recommend that you do the same for your implementation and return a usable +frame object instead of a raw byte array. + +Write frame, block and field classes +------------------------------------ + +BOF's core source code assumes that the network protocols transmit frames as +bytes arrays, which contain blocks, which contain fields. If they don't, you can +skip this part. Otherwise, your protocol implementation should include three +classes, inheriting from ``BOFFrame``, ``BOFBlock`` and ``BOFField``. + +Formats and behavior that do not match with what is decribed above (mostly, JSON +specification file organization) have to be written to your protocol +implementation's subclasses. + +.. note:: + + So far (BOF v0.2.X), part of the code that we expect to be generic and used + by most of the implementation is not written to BOF core, but to the KNX + implementation. We are carefully moving them as we notice that they can be + reused, but this is a long process and we don't want to miss steps. So far, + please refer to KNX's frame, block and field implementations in + ``bof/knx/knxframe.py`` to write your own implementation and feel free to try + and move part of it to the core (``bof/frame.py``). diff --git a/docs/man/knx.rst b/docs/man/knx.rst index eb196d6..130d405 100644 --- a/docs/man/knx.rst +++ b/docs/man/knx.rst @@ -186,6 +186,15 @@ constructor. existing_frame = knx.KnxFrame(type="DESCRIPTION REQUEST") received_frame = knx.KnxFrame(bytes=data) +.. warning:: For some frames, the structure of the body depends on the value of + a field inside the body, and sometimes inside the same block. Therefore, we + have to specify the value for that field as soon as possible. When the frame + is built from a received byte array, this part is handled directly, but when + building a frame from the specification, please remember to set this value in + the constructor:: + + KnxFrame(type="CONNECT REQUEST", connection_type_code="Tunneling connection") + From the specification ++++++++++++++++++++++ @@ -277,45 +286,70 @@ available yet) contains details on how to do so. print(frame) -(TODO) Modify frames --------------------- +Modify frames +------------- -Say we want to create a ``CONNECT REQUEST`` frame. Using the two previous -sections, here is how to do it. +Modify fields ++++++++++++++ -We first need information associated to the current connection (source ip and -port, stored within a ``KnxNet`` object after the UDP connection is -established). +As explained previously, blocks and fields are attributes within a frame or +block object and can be reached using a syntax such as:: -.. code-block:: python + request.body.communication_channel_id - ip, port = knxnet.source +.. tip:: Use ``print(request)`` and ``request.fields`` to locate the fields + you need to change. -We have to create the predefined frame with the appropriate format (blocks and -fields), but some of them are empty (values set to 0). We then have to fill some -of them that are required to be understood by the server. +Terminal parts of a frame are KnxField objects, when you want to modify field +values, you need to refer to the field's attributes:: -.. code-block:: python + request.body.communication_channel_id.value = "test" - connectreq = knx.KnxFrame(type="CONNECT REQUEST") +Value accepts different types of values, which will be converted to bytes: +``str``, ``int`` and ``bytes``. ``str`` with the following format have a +different conversion strategy; - connectreq.body.control_endpoint.ip_address.value = ip - connectreq.body.control_endpoint.port.value = port - connectreq.body.data_endpoint.ip_address.value = ip - connectreq.body.data_endpoint.port.value = port +:IPv4: ``A.B.C.D`` is converted to 4 corresponding bytes. +:KNX address: *Not implemented yet* -Finally, we need to specify the type of connection we want to establish with the -server. The latter is sent as an identifier in the field -``connection_type_code``. The list of existing identifiers for this field are -defined in the KNX standard and reported to the JSON specification -file. Therefore, we can either set the ID manually, or refer to the -specification file. The content of the specification file can be accessed by -instantiating the singleton class ``KnxSpec``. From this object, the sections in -the JSON file can be accessed as properties (details in "Advanced Usage" (not -available yet)). +Values you set change the size (the total size is recalculated anyway) of the +field if they do not match. You may need to resize manually, e.g. with +``byte.resize()``. Else you can set the size yourself beforehand:: -.. code-block:: + request.body.communication_channel_id.size = 1 + +When you do so, the size parameter switches to a "manual" mode, and will not +change until the end user manually changes it. + +Modify blocks (and frames) +++++++++++++++++++++++++++ + +Blocks order, types and names are based on the JSON specification file +``knxnet.json``, which has been written according to KNX Standard v2.1. That +said, please note that we had to made some very small adaptations, and that some +types of messages, blocks and codes are still missing. + +There are two ways to modify blocks within a frame: + +:From the objects: + + Blocks in a frame can be added, removed, or changed using ``append`` and + ``remove`` and by manipulating directly ``KnxBlock`` objects. For instance:: + + block = knx.KnxBlock(name="atoll") + block.append(knx.KnxField(name="pom-")) + block.append(knx.KnxField(name="pom")) `` + +:From the specification file: + + You can obviously replace, change or extend the specification file. BOF + should not comply, unless the JSON parser can't read it, or unless it does + not contain the 3 required sections ``frame``, ``blocks`` and ``codes``. + Please refer to section "Extend BOF" for more information. + +.. warning:: - knxspecs = knx.KnxSpec() - connection_type = knxspecs.connection_types["Device Management Connection"] - connectreq.body.connection_request_information.connection_type_code.value = connection_type + KNX frame servers usually have strict parsing rules and won't consider + invalid frames. If you modify the structure of a frame or block and differ + too much from the specification, you should not expect the KNX device to + respond. From cc4a1bd0c6a14eae93517a593751de6c4afead0b Mon Sep 17 00:00:00 2001 From: JulienBedel Date: Mon, 3 Aug 2020 11:42:17 +0200 Subject: [PATCH 09/24] feat(opcua): Add suport for dependency search in raw bytes - Now 2 ways to search for dependencies when creating blocks, either using "default" parameter or with raw byte value - Impliede defining get_code_names in OpcuaSpec (instead of get_association that was used before) - Add tests for dependency search - Add new examples in OpcuaBlock docstring --- bof/opcua/opcuaframe.py | 126 +++++++++++++++++++++++--------------- tests/test_opcua_frame.py | 62 ++++++++++++++----- 2 files changed, 121 insertions(+), 67 deletions(-) diff --git a/bof/opcua/opcuaframe.py b/bof/opcua/opcuaframe.py index d0bd897..ae02e5b 100644 --- a/bof/opcua/opcuaframe.py +++ b/bof/opcua/opcuaframe.py @@ -36,7 +36,7 @@ class OpcuaSpec(BOFSpec): spec = opcua.OpcuaSpec() block_template = spec.get_block_template(block_name="HEL_BODY") item_template = spec.get_item_template("HEL_BODY", "protocol version") - message_structure = spec.get_association("message_type", "HEL") + message_structure = spec.get_code_name("message_type", "HEL") """ def __init__(self): @@ -74,21 +74,25 @@ def get_block_template(self, block_name:str) -> list: :param block_name: Name of the block we want the template from. """ return self._get_dict_value(self.blocks, block_name) if block_name else None - - def get_association(self, code_name:str, identifier:str) -> str: + + def get_code_name(self, code_name:str, identifier) -> str: """Returns the value associated to an `identifier` inside a `code_name` association table. See `opcua.json` + usage example to better understand the association table concept. - :param identifier: Key we want the value from. - :code name: Association table name we want to look into for identifier + :param identifier: Key we want the value from, as str or byte. + :param code_name: Association table name we want to look into for identifier match. """ - #TODO: add support for bytes codes names (if needed in the specs ?) - if code_name in self.codes: - for association in self.codes[code_name]: - if identifier == association: - return self.codes[code_name][association] + code_name = self._get_dict_key(self.codes, code_name) + if isinstance(identifier, bytes): + for key in self.codes[code_name]: + if identifier == str.encode(key): + return self.codes[code_name][key] + if isinstance(identifier, str): + for key in self.codes[code_name]: + if identifier == key: + return self.codes[code_name][key] return None ############################################################################### @@ -137,31 +141,35 @@ class OpcuaBlock(BOFBlock): value of another field. In that case the keyword "depends:" is used to associate the variable to its value, based on given parameters to another field. The process can be looked in details `__init__` method, and - understood from examples. They are not perfect/exhaustive (but at least allowed to identify some bugs when coding). + understood from examples. Usage example:: - # block creation directly from type - block = opcua.OpcuaBlock(name="parent-block", type="HEADER") + # block creation with direct parameters + block = opcua.OpcuaBlock(name="header", type="HEADER") - # block creation from an item template + # block creation from an item template (as found in json spec file) item_template_block = {"name": "header", "type": "HEADER"} block = opcua.OpcuaBlock(item_template_block=item_template_block) - # block creation from an item template with dependency (defaults is needed) - # see opcua.json to understand defaults parameter value. + # block creation from an item template with dependency specified in defaults + # parameters item_template_block = {"name": "body", "type": "depends:message_type"} block = opcua.OpcuaBlock( item_template_block={"name": "body", "type": "depends:message_type"}, defaults={"message_type": "HEL"}) - # fills block with value at creation (a type is still mandatory) + # fills block with byte value at creation (note that a type is still mandatory) block = opcua.OpcuaBlock(type="STRING", value=14*b"\x01") - # block customization "by hand" - block = opcua.OpcuaBlock(type="HEADER") - sub_block = opcua.OpcuaBlock(type="STRING") - block.append(sub_block) + # a block with dependency can be created from raw bytes too (no default parameter) + data1 = b'HEL\x00...' + data2 = b'\x00...' + block = opcua.OpcuaBlock() + block.append(opcua.OpcuaBlock(value=data1, parent=block, + **{"name": "header", "type": "HEADER"})) + block.append(opcua.OpcuaBlock(alue=data2, parent=block, + **{"name": "body", "type": "depends:message_type"})) # we can access list block `fields` using block.attributes @@ -179,15 +187,20 @@ def factory(cls, item_template:dict, **kwargs) -> object: Keyword arguments: - :param defaults: defaults values in a dict, needed to construct - blocks with dependencies, see example above. + :param defaults: defaults values to assign a field as dictionnary + (can therefore be used to construct blocks with + dependencies if not found in raw bytes, see example + above) :param value: bytes value to fill the block with - + (can create dependencies on its own, see example + above) """ # case where item template represents a field (non-recursive) if "type" in item_template and item_template["type"] == "field": value = b'' - if "value" in kwargs and kwargs["value"]: + if "defaults" in kwargs and item_template["name"] in kwargs["defaults"]: + value = kwargs["defaults"][template["name"]] + elif "value" in kwargs and kwargs["value"]: value = kwargs["value"][:item_template["size"]] return OpcuaField(**item_template, value=value) # case where item template represents a sub-block (nested/recursive block) @@ -202,13 +215,21 @@ def __init__(self, **kwargs): :param type: a string specifying block type (as found in json specifications) to construct the block on. :param item_template_block: item template dict corresponding to a - block (which is described in item_block_template['type']), - giving its structure to the OpcuaBlock. If a block type - is already specified this parameter won't be taken into - account. - :param defaults: defaults values in a dict, needed to construct - blocks with dependencies. + block (which is described in item_block_template['type']), + giving its structure to the OpcuaBlock. If a block type + is already specified this parameter won't be taken into + account. + :param defaults: defaults values to assign a field as dictionnary + (can therefore be used to construct blocks with + dependencies if not found in raw bytes, see example + above) :param value: bytes value to fill the block with + (can create dependencies on its own, see example + above). If defaults parameter is found it overcomes + the value passed as bytes. + + See example in class docstring to understand dependency creation + either with defaults or with value parameter. """ self._spec = OpcuaSpec() @@ -217,7 +238,6 @@ def __init__(self, **kwargs): # we gather args values and set some default values first defaults = kwargs["defaults"] if "defaults" in kwargs else {} value = kwargs["value"] if "value" in kwargs else {} - block_type = None block_template = None @@ -235,13 +255,19 @@ def __init__(self, **kwargs): # if a dependency is found in block type # looks for needed information in default arg if block_type.startswith("depends:"): - dependency = to_property(block_type.split(":")[1]) + dependency = block_type.split(":")[1] + + # checks first for dependency defaults parameter if dependency in defaults: - block_type = self._spec.get_association(dependency, defaults[dependency]) + block_type = self._spec.get_code_name(dependency, defaults[dependency]) + if dependency == None: + raise BOFProgrammingError("No valid association found for dependency '{0}' with '{1}'.".format(dependency, defaults[dependency])) + # if not found in defaults, check in parent block else: - raise BOFProgrammingError("Dependecy '{0}' missing in defaults parameter".format(dependency)) - if not block_type: - raise BOFProgrammingError("Association not found for dependency '{0}'".format(dependency)) + block_type = self._get_depends_block(dependency) + if block_type == None: + raise BOFProgrammingError("No valid association found in parents for dependency '{0}'.".format(dependency)) + log("Creating OpcuaBlock of type '{0}' from dependency '{1}'.".format(block_type, dependency)) else: log("Creating OpcuaBlock of type '{0}'.".format(block_type)) @@ -249,20 +275,20 @@ def __init__(self, **kwargs): # for the moment, we set the block name to its type self._name = block_type - # if block template has not been we extract the block template according to the type found in item template + # if block template has not been found we extract the block template according to the type found in item template block_template = self._spec.get_block_template(block_type) - if block_template: - for item_template in block_template: - new_item = self.factory(item_template, defaults=defaults, value=value, parent=self) - self.append(new_item) - - # Cut value to fill byte by byte - if value: - if len(new_item) >= len(value): - break - value = value[len(new_item):] - else: + if not block_template: raise BOFProgrammingError("Block type '{0}' not found in specifications.".format(block_type)) - return + for item_template in block_template: + new_item = self.factory(item_template, defaults=defaults, value=value, parent=self) + self.append(new_item) + + # Cut value to fill byte by byte + if value: + if len(new_item) >= len(value): + break + value = value[len(new_item):] + + return \ No newline at end of file diff --git a/tests/test_opcua_frame.py b/tests/test_opcua_frame.py index ba7e828..8489bf1 100644 --- a/tests/test_opcua_frame.py +++ b/tests/test_opcua_frame.py @@ -60,17 +60,28 @@ def test_09_opcua_spec_get_item_invalid(): spec = opcua.OpcuaSpec("./jsons/valid_opcua.json") item_template = spec.get_template("INVALID", "INVALID") self.assertEqual(item_template, None) - def test_10_get_association_valid(): + def test_10_get_association_str_valid(): """Test that a valid association is returned as expeted""" spec = opcua.OpcuaSpec("./jsons/valid_opcua.json") - message_structure = spec.get_association("message_type", "HEL") + message_structure = spec.get_code_name("message_type", "HEL") self.assertEqual(message_structure, "HEL_BODY") - def test_11_get_association_invalid(): - """Test that an ivalid association returns None""" + def test_11_get_association_str_invalid(): + """Test that an invalid association returns None""" spec = opcua.OpcuaSpec("./jsons/valid_opcua.json") - message_structure = spec.get_association("INVALID", "INVALID") + message_structure = spec.get_code_name("INVALID", "INVALID") + self.assertEqual(message_structure, None) + def test_12_get_association_bytes_valid(): + """Test that a valid association with byte id is returned as expeted""" + spec = opcua.OpcuaSpec("./jsons/valid_opcua.json") + message_structure = spec.get_code_name("message_type", b"HEL") + self.assertEqual(message_structure, "HEL_BODY") + def test_13_get_association_bytes_invalid(): + """Test that an invalid association with byte id returns None""" + spec = opcua.OpcuaSpec("./jsons/valid_opcua.json") + message_structure = spec.get_code_name("INVALID", b"INVALID") self.assertEqual(message_structure, None) + class Test02OpcuaField(unittest.TestCase): """Test class for field crafting and access. Note that we don't test for the whole BOFField behavior, but @@ -140,31 +151,48 @@ def test_07_opcua_create_nested_block(self): sub_block = opcua.OpcuaBlock(type="STRING") block.append(sub_block) self.assertEqual(block.string.attributes, ['string_length', 'string_value']) - def test_08_opcua_create_dependency_block(self): - """Test dependecy block creation""" + def test_08_opcua_create_dependency_block_missing(self): + """Test nested block creation with missing dependency values (neither from + defaults or parent block""" + with self.assertRaises(BOFProgrammingError): + item_template_block = {"name": "body", "type": "depends:message_type"} + block = opcua.OpcuaBlock( + item_template_block={"name": "body", "type": "depends:message_type"}) + def test_09_opcua_create_dependency_defaults(self): + """Test dependecy block creation with defaults""" item_template_block = {"name": "body", "type": "depends:message_type"} block = opcua.OpcuaBlock( item_template_block={"name": "body", "type": "depends:message_type"}, defaults={"message_type": "HEL"}) self.assertEqual(block.name, "HEL_BODY") - def test_09_opcua_create_dependency_block_missing(self): - """Test nested block creation with missing defaults values""" - with self.assertRaises(BOFProgrammingError): - item_template_block = {"name": "body", "type": "depends:message_type"} - block = opcua.OpcuaBlock( - item_template_block={"name": "body", "type": "depends:message_type"}) - def test_10_opcua_create_nested_dependecy_block_wrong(self): + def test_10_opcua_create_dependency_defaults_wrong(self): """Test nested block creation with wrong defaults values""" with self.assertRaises(BOFProgrammingError): item_template_block = {"name": "body", "type": "depends:message_type"} block = opcua.OpcuaBlock( item_template_block={"name": "body", "type": "depends:message_type"}, defaults={"message_type": "unknown"}) - def test_11_opcua_create_block_with_value(self): + def test_11_opcua_create_block_value(self): """Test block creation with a value to fill it""" block = opcua.OpcuaBlock(type="STRING", value=14*b"\x01") self.assertEqual(block.string_length.value, b'\x01\x01\x01\x01') - def test_12_opcua_create_block_with_value_missing_type(self): + def test_12_opcua_create_block_value_missing_type(self): """Test that a block value with not type returns an empty list as expected""" block = opcua.OpcuaBlock() - self.assertEqual(block.content, []) \ No newline at end of file + self.assertEqual(block.content, []) + def test_13_opcua_create_dependency_bytes(self): + """Test nested block creation from raw bytes""" + data1 = b'HEL\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' + data2 = b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' + block = opcua.OpcuaBlock() + block.append(opcua.OpcuaBlock(value=data1, parent=block, **{"name": "header", "type": "HEADER"})) + block.append(opcua.OpcuaBlock(value=data2, parent=block, **{"name": "body", "type": "depends:message_type"})) + self.assertNotEqual(block.hel_body, None) + def test_14_opcua_create_dependency_bytes_wrong(self): + """Test nested block creation from raw bytes with wrong dependency""" + with self.assertRaises(BOFProgrammingError): + data1 = b'XYZ\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' + data2 = b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' + block = opcua.OpcuaBlock() + block.append(opcua.OpcuaBlock(value=data1, parent=block, **{"name": "header", "type": "HEADER"})) + block.append(opcua.OpcuaBlock(value=data2, parent=block, **{"name": "body", "type": "depends:message_type"})) \ No newline at end of file From 3d7a665e91472f30ee337e33a9be708badbba577 Mon Sep 17 00:00:00 2001 From: JulienBedel Date: Mon, 3 Aug 2020 11:49:29 +0200 Subject: [PATCH 10/24] fix(bof): Add case where _get_depends_block() is called with no parent In that case, returns None. --- bof/frame.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/bof/frame.py b/bof/frame.py index b9ea88d..6441d51 100644 --- a/bof/frame.py +++ b/bof/frame.py @@ -421,13 +421,16 @@ def _get_depends_block(self, field:str, defaults:dict=None): :raises BOFProgrammingError: If specified field was not found. """ field = to_property(field) - field_list = list(self._parent) - field_list.reverse() - for frame_field in field_list: - if field == to_property(frame_field.name): - block = self._spec.get_code_name(frame_field.name, frame_field.value) - return block - raise BOFProgrammingError("Field not found ({0}).".format(field)) + if self._parent: + field_list = list(self._parent) + field_list.reverse() + for frame_field in field_list: + if field == to_property(frame_field.name): + block = self._spec.get_code_name(frame_field.name, frame_field.value) + return block + raise BOFProgrammingError("Field not found ({0}).".format(field)) + else: + return None #-------------------------------------------------------------------------# # Properties # From c8f76de46122a0e8ca79da4a1d2fcdf4bdbce025 Mon Sep 17 00:00:00 2001 From: JulienBedel Date: Tue, 4 Aug 2020 11:15:56 +0200 Subject: [PATCH 11/24] refactor(opcua): opcua module rewrite to match knx for genericity - Suspended development of new feature for refactoring with objective to gain more genericity in BOF objects. - As a result most of knxframe.py can be re-used in opcuaframe.py, hence it should be moved to frame.py - Rewrtote tests to take refactoring into account --- bof/opcua/opcua.json | 4 +- bof/opcua/opcuaframe.py | 136 ++++++++++++++++---------------------- tests/test_opcua_frame.py | 52 ++++----------- 3 files changed, 72 insertions(+), 120 deletions(-) diff --git a/bof/opcua/opcua.json b/bof/opcua/opcua.json index 7afe3d4..db82326 100644 --- a/bof/opcua/opcua.json +++ b/bof/opcua/opcua.json @@ -15,7 +15,7 @@ {"name": "send_buffer_size", "type": "field", "size": 4}, {"name": "max_message_size", "type": "field", "size": 4}, {"name": "max_chunk_count", "type": "field", "size": 4}, - {"name": "endpoint_url", "type": "STRING", "size": 18} + {"name": "endpoint_url", "type": "STRING"} ], "STRING": [ {"name": "string_length", "type": "field", "size": 4}, @@ -27,4 +27,4 @@ "HEL": "HEL_BODY" } } -} +} \ No newline at end of file diff --git a/bof/opcua/opcuaframe.py b/bof/opcua/opcuaframe.py index ae02e5b..ca1bc75 100644 --- a/bof/opcua/opcuaframe.py +++ b/bof/opcua/opcuaframe.py @@ -34,7 +34,7 @@ class OpcuaSpec(BOFSpec): Usage example:: spec = opcua.OpcuaSpec() - block_template = spec.get_block_template(block_name="HEL_BODY") + block_template = spec.get_block_template("HEL_BODY") item_template = spec.get_item_template("HEL_BODY", "protocol version") message_structure = spec.get_code_name("message_type", "HEL") """ @@ -61,38 +61,41 @@ def get_item_template(self, block_name:str, item_name:str) -> dict: :param block_name: Name of the block containing the item. :param item_name: Name of the item to look for in the block. + :returns: Item template associated to block_name and item_name """ if block_name in self.blocks: for item in self.blocks[block_name]: if item['name'] == item_name: return item return None - + def get_block_template(self, block_name:str) -> list: """Returns a block template (list of item templates) from a block name. :param block_name: Name of the block we want the template from. + :returns: Block template associated with the specifified block_name. """ return self._get_dict_value(self.blocks, block_name) if block_name else None - def get_code_name(self, code_name:str, identifier) -> str: - """Returns the value associated to an `identifier` inside a `code_name` - association table. See `opcua.json` + usage example to better - understand the association table concept. + def get_code_name(self, code:str, identifier) -> str: + """Returns the value associated to an `identifier` inside a + `code` association table. See opcua.json + usage + example to better understand the association table concept. :param identifier: Key we want the value from, as str or byte. - :param code_name: Association table name we want to look into for identifier - match. + :param code: Association table name we want to look into + for identifier match. + :returns: value associated to an identifier inside a code. """ - code_name = self._get_dict_key(self.codes, code_name) + code = self._get_dict_key(self.codes, code) if isinstance(identifier, bytes): - for key in self.codes[code_name]: + for key in self.codes[code]: if identifier == str.encode(key): - return self.codes[code_name][key] - if isinstance(identifier, str): - for key in self.codes[code_name]: + return self.codes[code][key] + elif isinstance(identifier, str): + for key in self.codes[code]: if identifier == key: - return self.codes[code_name][key] + return self.codes[code][key] return None ############################################################################### @@ -105,17 +108,19 @@ def get_code_name(self, code_name:str, identifier) -> str: class OpcuaField(BOFField): """An ``OpcuaField`` is a set of raw bytes with a name, a size and a - content (``value``). Inherits ``BOFField``. See `frame.py`. + content (``value``). Inherits ``BOFField``, see `bof/frame.py`. Usage example:: - # creating a field from raw parameters + # creating a field from parameters field = opcua.OpcuaField(name="protocol version", value=b"1", size=4) - field.value = field.value = b'\x00\x00\x00\x02' # creating a field from a template item_template_field = spec.get_item_template("HEL_BODY", "protocol version") field = opcua.OpcuaField(**item_template_field) + + # editing field value + field.value = b'\x00\x00\x00\x02' """ # For now there is no need to redefine getter and setter property from @@ -127,42 +132,41 @@ class OpcuaField(BOFField): class OpcuaBlock(BOFBlock): """Object representation of an OPC UA block. Inherits ``BOFBlock``. - See `frame.py`. + See `bof/frame.py`. An OpcuaBlock (as well as a BOFBlock) contains a set of items. Those items can be fields as well as blocks, therefore creating - "nested blocks" or "sub-block". + so-called "nested blocks" (or "sub-block"). - It is usually built from a template which gives its structure. - Values can also be specified to "fill" the block stucture. + A block is usually built from a template which gives its structure. + Bytes can also be specified to "fill" this block stucture. If the bytes + values are coherent, the structure can also be determined directly from + them. If no structure is specified the block remains empty. - Some block field's value (typically a sub-block type) may depend on the + Some block field value (typically a sub-block type) may depend on the value of another field. In that case the keyword "depends:" is used to associate the variable to its value, based on given parameters to another - field. The process can be looked in details `__init__` method, and + field. The process can be looked in details in the `__init__` method, and understood from examples. Usage example:: + # empty block creation + block = opcua.OpcuaBlock(name="empty_block") + # block creation with direct parameters block = opcua.OpcuaBlock(name="header", type="HEADER") # block creation from an item template (as found in json spec file) item_template_block = {"name": "header", "type": "HEADER"} - block = opcua.OpcuaBlock(item_template_block=item_template_block) - - # block creation from an item template with dependency specified in defaults - # parameters - item_template_block = {"name": "body", "type": "depends:message_type"} - block = opcua.OpcuaBlock( - item_template_block={"name": "body", "type": "depends:message_type"}, - defaults={"message_type": "HEL"}) + block = opcua.OpcuaBlock(**item_template_block) - # fills block with byte value at creation (note that a type is still mandatory) + # fills block with byte value at creation + (note that here a type is still mandatory) block = opcua.OpcuaBlock(type="STRING", value=14*b"\x01") - # a block with dependency can be created from raw bytes too (no default parameter) + # a block with dependency can be created from raw bytes (no default parameter) data1 = b'HEL\x00...' data2 = b'\x00...' block = opcua.OpcuaBlock() @@ -170,30 +174,27 @@ class OpcuaBlock(BOFBlock): **{"name": "header", "type": "HEADER"})) block.append(opcua.OpcuaBlock(alue=data2, parent=block, **{"name": "body", "type": "depends:message_type"})) + # this in an example of block looking in its sibling for dependency value - # we can access list block `fields` using + # we can access a list of available block `fields` using : block.attributes - # and access on of those `fields` with - block.protocol_version + # and access one of those `fields` with : + block.field_name """ - + @classmethod def factory(cls, item_template:dict, **kwargs) -> object: """Returns either an `OpcuaBlock` or an `OpcuaField` depending on the template specified item type. That's why it's a factory as a class method. :param item_template: item template representing sub-block or field. + :returns: A new instance of an OpcuaBlock or an OpcuaField. Keyword arguments: - :param defaults: defaults values to assign a field as dictionnary - (can therefore be used to construct blocks with - dependencies if not found in raw bytes, see example - above) - :param value: bytes value to fill the block with - (can create dependencies on its own, see example - above) + :param defaults: Defaults values to assign a field as dictionnary. + :param value: Bytes value to fill the item (block or field) with. """ # case where item template represents a field (non-recursive) if "type" in item_template and item_template["type"] == "field": @@ -205,7 +206,7 @@ def factory(cls, item_template:dict, **kwargs) -> object: return OpcuaField(**item_template, value=value) # case where item template represents a sub-block (nested/recursive block) else: - return OpcuaBlock(item_template_block=item_template, **kwargs) + return cls(**item_template, **kwargs) def __init__(self, **kwargs): """Initialize the ``OpcuaBlock``. @@ -214,11 +215,6 @@ def __init__(self, **kwargs): :param type: a string specifying block type (as found in json specifications) to construct the block on. - :param item_template_block: item template dict corresponding to a - block (which is described in item_block_template['type']), - giving its structure to the OpcuaBlock. If a block type - is already specified this parameter won't be taken into - account. :param defaults: defaults values to assign a field as dictionnary (can therefore be used to construct blocks with dependencies if not found in raw bytes, see example @@ -238,51 +234,35 @@ def __init__(self, **kwargs): # we gather args values and set some default values first defaults = kwargs["defaults"] if "defaults" in kwargs else {} value = kwargs["value"] if "value" in kwargs else {} - block_type = None - block_template = None + self._name = kwargs["name"] if "name" in kwargs else "" - # there are several way to get a block type, either by its name - # or by specifying a template - if "type" in kwargs: + # we gather the block type from args, if not found then returns an + # empty block + if "type" in kwargs and kwargs["type"] != "block": block_type = kwargs["type"] - elif "item_template_block" in kwargs and "type" in kwargs["item_template_block"]: - item_template_block = kwargs["item_template_block"] - block_type = item_template_block["type"] else: log("No type or item_template_block specified, creating empty block") return - + # if a dependency is found in block type # looks for needed information in default arg if block_type.startswith("depends:"): dependency = block_type.split(":")[1] - - # checks first for dependency defaults parameter - if dependency in defaults: - block_type = self._spec.get_code_name(dependency, defaults[dependency]) - if dependency == None: - raise BOFProgrammingError("No valid association found for dependency '{0}' with '{1}'.".format(dependency, defaults[dependency])) - # if not found in defaults, check in parent block - else: - block_type = self._get_depends_block(dependency) - if block_type == None: - raise BOFProgrammingError("No valid association found in parents for dependency '{0}'.".format(dependency)) - + block_type = self._get_depends_block(dependency) + if not block_type: + raise BOFProgrammingError("No valid association found in parents for dependency '{0}'.".format(dependency)) log("Creating OpcuaBlock of type '{0}' from dependency '{1}'.".format(block_type, dependency)) else: log("Creating OpcuaBlock of type '{0}'.".format(block_type)) - - # for the moment, we set the block name to its type - self._name = block_type - - # if block template has not been found we extract the block template according to the type found in item template + + # we extract the block template according to its type block_template = self._spec.get_block_template(block_type) if not block_template: raise BOFProgrammingError("Block type '{0}' not found in specifications.".format(block_type)) for item_template in block_template: - new_item = self.factory(item_template, defaults=defaults, value=value, parent=self) + new_item = self.factory(item_template, value=value, defaults=defaults, parent=self) self.append(new_item) # Cut value to fill byte by byte @@ -290,5 +270,3 @@ def __init__(self, **kwargs): if len(new_item) >= len(value): break value = value[len(new_item):] - - return \ No newline at end of file diff --git a/tests/test_opcua_frame.py b/tests/test_opcua_frame.py index 8489bf1..79a9563 100644 --- a/tests/test_opcua_frame.py +++ b/tests/test_opcua_frame.py @@ -123,13 +123,13 @@ def test_01_opcua_create_block_empty(self): def test_02_opcua_create_block_template(self): """Tests that a block can be created as expected from a template""" item_template_block = {"name": "header", "type": "HEADER"} - block = opcua.OpcuaBlock(item_template_block=item_template_block) + block = opcua.OpcuaBlock(**item_template_block) self.assertEqual(block.content[0].name, "message_type") def test_03_opcua_create_block_template_invalid(self): """Tests that block creation with invalid template fail case is handled""" with self.assertRaises(BOFProgrammingError): item_template_block = {"name": "header", "type": "unknown"} - block = opcua.OpcuaBlock(item_template_block=item_template_block) + block = opcua.OpcuaBlock(**item_template_block) def test_04_opcua_create_block_type(self): """Tests that a block can be created as expected from a type name""" block = opcua.OpcuaBlock(type="HEADER") @@ -138,61 +138,35 @@ def test_05_opcua_create_block_type_invalid(self): """Tests that block creation with invalid type name fail case is handled""" with self.assertRaises(BOFProgrammingError): block = opcua.OpcuaBlock(type="unknown") - def test_06_opcua_create_mixed(self): - """Tests that in case of mixed block type specification - (type + template) only the type is kept""" - item_template_block = {"name": "header", "type": "unknown"} - block = opcua.OpcuaBlock(item_template_block=item_template_block, type="STRING") - self.assertEqual(block.content[0].name, "string_length") - self.assertNotEqual(block.content[0].name, "message_type") - def test_07_opcua_create_nested_block(self): + def test_06_opcua_create_nested_block(self): """Test for manual creation of nested block""" - block = opcua.OpcuaBlock(type="HEADER") - sub_block = opcua.OpcuaBlock(type="STRING") + block = opcua.OpcuaBlock(name="header", type="HEADER") + sub_block = opcua.OpcuaBlock(name="sub_block", type="STRING", parent=block) block.append(sub_block) - self.assertEqual(block.string.attributes, ['string_length', 'string_value']) - def test_08_opcua_create_dependency_block_missing(self): + self.assertEqual(block.sub_block.attributes, ['string_length', 'string_value']) + def test_07_opcua_create_dependency_block_missing(self): """Test nested block creation with missing dependency values (neither from defaults or parent block""" with self.assertRaises(BOFProgrammingError): item_template_block = {"name": "body", "type": "depends:message_type"} - block = opcua.OpcuaBlock( - item_template_block={"name": "body", "type": "depends:message_type"}) - def test_09_opcua_create_dependency_defaults(self): - """Test dependecy block creation with defaults""" - item_template_block = {"name": "body", "type": "depends:message_type"} - block = opcua.OpcuaBlock( - item_template_block={"name": "body", "type": "depends:message_type"}, - defaults={"message_type": "HEL"}) - self.assertEqual(block.name, "HEL_BODY") - def test_10_opcua_create_dependency_defaults_wrong(self): - """Test nested block creation with wrong defaults values""" - with self.assertRaises(BOFProgrammingError): - item_template_block = {"name": "body", "type": "depends:message_type"} - block = opcua.OpcuaBlock( - item_template_block={"name": "body", "type": "depends:message_type"}, - defaults={"message_type": "unknown"}) - def test_11_opcua_create_block_value(self): + block = opcua.OpcuaBlock(**{"name": "body", "type": "depends:message_type"}) + def test_08_opcua_create_block_value(self): """Test block creation with a value to fill it""" block = opcua.OpcuaBlock(type="STRING", value=14*b"\x01") self.assertEqual(block.string_length.value, b'\x01\x01\x01\x01') - def test_12_opcua_create_block_value_missing_type(self): - """Test that a block value with not type returns an empty list as expected""" - block = opcua.OpcuaBlock() - self.assertEqual(block.content, []) - def test_13_opcua_create_dependency_bytes(self): + def test_09_opcua_create_dependency_bytes(self): """Test nested block creation from raw bytes""" data1 = b'HEL\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' data2 = b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' block = opcua.OpcuaBlock() block.append(opcua.OpcuaBlock(value=data1, parent=block, **{"name": "header", "type": "HEADER"})) block.append(opcua.OpcuaBlock(value=data2, parent=block, **{"name": "body", "type": "depends:message_type"})) - self.assertNotEqual(block.hel_body, None) - def test_14_opcua_create_dependency_bytes_wrong(self): + self.assertNotEqual(block.body, None) + def test_10_opcua_create_dependency_bytes_wrong(self): """Test nested block creation from raw bytes with wrong dependency""" with self.assertRaises(BOFProgrammingError): data1 = b'XYZ\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' data2 = b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' block = opcua.OpcuaBlock() block.append(opcua.OpcuaBlock(value=data1, parent=block, **{"name": "header", "type": "HEADER"})) - block.append(opcua.OpcuaBlock(value=data2, parent=block, **{"name": "body", "type": "depends:message_type"})) \ No newline at end of file + block.append(opcua.OpcuaBlock(value=data2, parent=block, **{"name": "body", "type": "depends:message_type"})) From 1773a5fcea01c3d8373e17a56d15789d9315364e Mon Sep 17 00:00:00 2001 From: JulienBedel Date: Tue, 4 Aug 2020 13:49:29 +0200 Subject: [PATCH 12/24] feat(opcua): Add BOFFrame implementation to opcua - Add OpcuaFrame class in opcua module to implement BOFFrame. - Particular care has been taken to stay close to KNX implementation. - Test have not been written yet, but basic usage has been verified with no anormal behavior. --- bof/opcua/opcuaframe.py | 118 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 117 insertions(+), 1 deletion(-) diff --git a/bof/opcua/opcuaframe.py b/bof/opcua/opcuaframe.py index ca1bc75..37ba45f 100644 --- a/bof/opcua/opcuaframe.py +++ b/bof/opcua/opcuaframe.py @@ -16,6 +16,7 @@ from ..base import BOFProgrammingError, to_property, log from ..frame import BOFFrame, BOFBlock, BOFField from ..spec import BOFSpec +from .. import byte ############################################################################### # OPCUA SPECIFICATION CONTENT # @@ -98,6 +99,30 @@ def get_code_name(self, code:str, identifier) -> str: return self.codes[code][key] return None + def get_code_id(self, dict_key:dict, name:str) -> bytes: + """TODO:""" + dict_key = self._get_dict_key(self.codes, dict_key) + for key, value in self.codes[dict_key].items(): + if name == value: + return bytes.fromhex(key) + return None + + def get_association(self, code_name:str, identifier) -> str: + """Returns the value associated to an `identifier` inside a `code_name` + association table. See `opcua.json` + usage example to better + understand the association table concept. + + :param identifier: Key we want the value from. + :code name: Association table name we want to look into for identifier + match. + """ + #TODO: add support for bytes codes names (if needed in the specs ?) + if code_name in self.codes: + for association in self.codes[code_name]: + if identifier == association: + return self.codes[code_name][association] + return None + ############################################################################### # OPC UA FRAME CONTENT # ############################################################################### @@ -200,7 +225,7 @@ def factory(cls, item_template:dict, **kwargs) -> object: if "type" in item_template and item_template["type"] == "field": value = b'' if "defaults" in kwargs and item_template["name"] in kwargs["defaults"]: - value = kwargs["defaults"][template["name"]] + value = kwargs["defaults"][item_template["name"]] elif "value" in kwargs and kwargs["value"]: value = kwargs["value"][:item_template["size"]] return OpcuaField(**item_template, value=value) @@ -270,3 +295,94 @@ def __init__(self, **kwargs): if len(new_item) >= len(value): break value = value[len(new_item):] + +#-----------------------------------------------------------------------------# +# OPC UA frames representation # +#-----------------------------------------------------------------------------# + +class OpcuaFrame(BOFFrame): + """Object representation of an OPC UA frame, created from the tempalte + in `opcua.json`. + + Uses various initialization methods to create a frame : + + :Byte array: Build the object from a raw byte array, typically used + when receiving incoming connection. In this case block + dependencies are identified automatically. + :Keyword arguments: Uses keyword described in __defaults to fill frame + fields. + + Usage example:: + + # creation from raw bytes (format is automatically identified) + data = b'HEL\x00..' + frame = opcua.OpcuaFrame(bytes=data) + + # creation from known type (who is actually a needed dependence) + # in order to create the frame (see frame structure in opcua.json) + frame = opcua.OpcuaFrame(type="MSG") + """ + + __defaults = { + # {Argument name: field name} + "type": "message_type", + } + + def __init__(self, **kwargs): + """Initialize an OpcuaFrame from various origins using values from + keyword arguments : + + Keyword arguments : + + :param byte: raw byte array used to build a frame. + :defaults arguments: every element of __default specifies arguments + that can be passed in order to set fields values + at frame creation. + """ + spec = OpcuaSpec() + super().__init__() + + # We store some values before starting building the frame + value = kwargs["bytes"] if "bytes" in kwargs else None + defaults = {} + for arg, code in self.__defaults.items(): + if arg in kwargs: + defaults[code] = str.encode(kwargs[arg]) + # Now we can start + for block in spec.frame: + # Create block + opcuablock = OpcuaBlock(value=value, defaults=defaults, parent=self, **block) + self.append(block["name"], opcuablock) + # If a value is used to fill the blocks, update it + if value: + if len(self._blocks[block["name"]]) >= len(value): + break + value = value[len(self._blocks[block["name"]]):] + # Update total frame length in header + self.update() + + #-------------------------------------------------------------------------# + # Public # + #-------------------------------------------------------------------------# + + def update(self): + """Update ``message_size`` field in header according to total + frame length. + """ + #super().update() + if "message_size" in self._blocks["header"].attributes: + total = sum([len(block) for block in self._blocks.values()]) + self._blocks["header"].message_size._update_value(byte.from_int(total)) + + #-------------------------------------------------------------------------# + # Properties # + #-------------------------------------------------------------------------# + + @property + def header(self): + self.update() + return self._blocks["header"] + @property + def body(self): + self.update() + return self._blocks["body"] \ No newline at end of file From ef75340438f8d0c5c7b6e0f91e6c2c36436ef7e7 Mon Sep 17 00:00:00 2001 From: JulienBedel Date: Tue, 4 Aug 2020 15:54:50 +0200 Subject: [PATCH 13/24] refactor(bof): Delete non-used method Delete non-used method from OpcuaSpec. --- bof/opcua/opcuaframe.py | 29 +---------------------------- 1 file changed, 1 insertion(+), 28 deletions(-) diff --git a/bof/opcua/opcuaframe.py b/bof/opcua/opcuaframe.py index 37ba45f..55a5562 100644 --- a/bof/opcua/opcuaframe.py +++ b/bof/opcua/opcuaframe.py @@ -99,30 +99,6 @@ def get_code_name(self, code:str, identifier) -> str: return self.codes[code][key] return None - def get_code_id(self, dict_key:dict, name:str) -> bytes: - """TODO:""" - dict_key = self._get_dict_key(self.codes, dict_key) - for key, value in self.codes[dict_key].items(): - if name == value: - return bytes.fromhex(key) - return None - - def get_association(self, code_name:str, identifier) -> str: - """Returns the value associated to an `identifier` inside a `code_name` - association table. See `opcua.json` + usage example to better - understand the association table concept. - - :param identifier: Key we want the value from. - :code name: Association table name we want to look into for identifier - match. - """ - #TODO: add support for bytes codes names (if needed in the specs ?) - if code_name in self.codes: - for association in self.codes[code_name]: - if identifier == association: - return self.codes[code_name][association] - return None - ############################################################################### # OPC UA FRAME CONTENT # ############################################################################### @@ -282,7 +258,6 @@ def __init__(self, **kwargs): # we extract the block template according to its type block_template = self._spec.get_block_template(block_type) - if not block_template: raise BOFProgrammingError("Block type '{0}' not found in specifications.".format(block_type)) @@ -320,7 +295,7 @@ class OpcuaFrame(BOFFrame): # creation from known type (who is actually a needed dependence) # in order to create the frame (see frame structure in opcua.json) - frame = opcua.OpcuaFrame(type="MSG") + frame = opcua.OpcuaFrame(type="HEL") """ __defaults = { @@ -331,8 +306,6 @@ class OpcuaFrame(BOFFrame): def __init__(self, **kwargs): """Initialize an OpcuaFrame from various origins using values from keyword arguments : - - Keyword arguments : :param byte: raw byte array used to build a frame. :defaults arguments: every element of __default specifies arguments From 234076dd676aa103d2b9d1b154c50469c3a7d7f3 Mon Sep 17 00:00:00 2001 From: JulienBedel Date: Tue, 4 Aug 2020 17:00:50 +0200 Subject: [PATCH 14/24] fix(opcua): Fix json spec file loading in tests + Fix method - Fix json spec file loading in Test01OpcuaSpec, now all pass as expected - Make get_code_name return None in case of invalid code specified --- bof/opcua/opcuaframe.py | 4 +-- tests/jsons/valid.json | 4 +-- tests/test_opcua_frame.py | 64 +++++++++++++++++++++++---------------- 3 files changed, 42 insertions(+), 30 deletions(-) diff --git a/bof/opcua/opcuaframe.py b/bof/opcua/opcuaframe.py index 55a5562..2a84b11 100644 --- a/bof/opcua/opcuaframe.py +++ b/bof/opcua/opcuaframe.py @@ -89,11 +89,11 @@ def get_code_name(self, code:str, identifier) -> str: :returns: value associated to an identifier inside a code. """ code = self._get_dict_key(self.codes, code) - if isinstance(identifier, bytes): + if isinstance(identifier, bytes) and code in self.codes: for key in self.codes[code]: if identifier == str.encode(key): return self.codes[code][key] - elif isinstance(identifier, str): + elif isinstance(identifier, str) and code in self.codes: for key in self.codes[code]: if identifier == key: return self.codes[code][key] diff --git a/tests/jsons/valid.json b/tests/jsons/valid.json index 7afe3d4..db82326 100644 --- a/tests/jsons/valid.json +++ b/tests/jsons/valid.json @@ -15,7 +15,7 @@ {"name": "send_buffer_size", "type": "field", "size": 4}, {"name": "max_message_size", "type": "field", "size": 4}, {"name": "max_chunk_count", "type": "field", "size": 4}, - {"name": "endpoint_url", "type": "STRING", "size": 18} + {"name": "endpoint_url", "type": "STRING"} ], "STRING": [ {"name": "string_length", "type": "field", "size": 4}, @@ -27,4 +27,4 @@ "HEL": "HEL_BODY" } } -} +} \ No newline at end of file diff --git a/tests/test_opcua_frame.py b/tests/test_opcua_frame.py index 79a9563..4b5bdc9 100644 --- a/tests/test_opcua_frame.py +++ b/tests/test_opcua_frame.py @@ -19,69 +19,81 @@ def test_01_opcua_spec_instantiate_default(self): def test_02_opcua_spec_instantiate_custom_valid_json(self): """Test that a custom and valid opcua spec file works as expected.""" try: - spec = opcua.OpcuaSpec("./jsons/valid_opcua.json") + spec = opcua.OpcuaSpec() + spec.load("tests/jsons/valid.json") except BOFLibraryError: self.fail("Valid json spec should not raise BOFLibraryError.") def test_03_opcua_spec_instantiate_custom_invalid_json(self): """Test that a custom and invalid opcua spec raises exception.""" with self.assertRaises(BOFLibraryError): - spec = opcua.OpcuaSpec("./jsons/invalid.json") + spec = opcua.OpcuaSpec() + spec.load("tests/jsons/invalid.json") def test_04_opcua_spec_instantiate_custom_invalid_path(self): """Test an invalid custom path raises exception.""" with self.assertRaises(BOFLibraryError): - spec = opcua.OpcuaSpec("./jsons/unexisting.json") - def test_05_opcua_spec_property_access_valid(): + spec = opcua.OpcuaSpec() + spec.load("tests/jsons/unexisting.json") + def test_05_opcua_spec_property_access_valid(self): """Test that a JSON element can be accessed as property""" - spec = opcua.OpcuaSpec("./jsons/valid_opcua.json") + spec = opcua.OpcuaSpec() + spec.load("tests/jsons/valid.json") frame_template = spec.frame self.assertEqual(frame_template[0]["name"], "header") - def test_05_opcua_spec_property_access_invalid(): + def test_06_opcua_spec_property_access_invalid(self): """Test that a unexisting JSON element can't be accessed as property""" - spec = opcua.OpcuaSpec("./jsons/valid_opcua.json") + spec = opcua.OpcuaSpec() + spec.load("tests/jsons/valid.json") with self.assertRaises(AttributeError): frame_template = spec.unexisting - def test_06_opcua_spec_get_blocks_valid(): + def test_07_opcua_spec_get_blocks_valid(self): """Test that we can get block from spec as expected""" - spec = opcua.OpcuaSpec("./jsons/valid_opcua.json") + spec = opcua.OpcuaSpec() + spec.load("tests/jsons/valid.json") block_template = spec.get_block_template(block_name="HEADER") self.assertEqual(block_template[0]["name"], "message_type") - def test_07_opcua_spec_get_blocks_invalid(): + def test_08_opcua_spec_get_blocks_invalid(self): """Test that an invalid block request returns None""" - spec = opcua.OpcuaSpec("./jsons/valid_opcua.json") + spec = opcua.OpcuaSpec() + spec.load("tests/jsons/valid.json") block_template = spec.get_block_template(block_name="INVALID") self.assertEqual(block_template, None) - def test_08_opcua_spec_get_item_valid(): + def test_09_opcua_spec_get_item_valid(self): """Test that we can get an item from spec as expected""" - spec = opcua.OpcuaSpec("./jsons/valid_opcua.json") - item_template = spec.get_template("HEL_BODY", "protocol_version") + spec = opcua.OpcuaSpec() + spec.load("tests/jsons/valid.json") + item_template = spec.get_item_template("HEL_BODY", "protocol_version") self.assertEqual(item_template["name"], "protocol_version") - def test_09_opcua_spec_get_item_invalid(): + def test_10_opcua_spec_get_item_invalid(self): """Test that an invalid item request returns None""" - spec = opcua.OpcuaSpec("./jsons/valid_opcua.json") - item_template = spec.get_template("INVALID", "INVALID") + spec = opcua.OpcuaSpec() + spec.load("tests/jsons/valid.json") + item_template = spec.get_item_template("INVALID", "INVALID") self.assertEqual(item_template, None) - def test_10_get_association_str_valid(): + def test_11_get_association_str_valid(self): """Test that a valid association is returned as expeted""" - spec = opcua.OpcuaSpec("./jsons/valid_opcua.json") + spec = opcua.OpcuaSpec() + spec.load("tests/jsons/valid.json") message_structure = spec.get_code_name("message_type", "HEL") self.assertEqual(message_structure, "HEL_BODY") - def test_11_get_association_str_invalid(): + def test_12_get_association_str_invalid(self): """Test that an invalid association returns None""" - spec = opcua.OpcuaSpec("./jsons/valid_opcua.json") + spec = opcua.OpcuaSpec() + spec.load("tests/jsons/valid.json") message_structure = spec.get_code_name("INVALID", "INVALID") self.assertEqual(message_structure, None) - def test_12_get_association_bytes_valid(): + def test_13_get_association_bytes_valid(self): """Test that a valid association with byte id is returned as expeted""" - spec = opcua.OpcuaSpec("./jsons/valid_opcua.json") + spec = opcua.OpcuaSpec() + spec.load("tests/jsons/valid.json") message_structure = spec.get_code_name("message_type", b"HEL") self.assertEqual(message_structure, "HEL_BODY") - def test_13_get_association_bytes_invalid(): + def test_14_get_association_bytes_invalid(self): """Test that an invalid association with byte id returns None""" - spec = opcua.OpcuaSpec("./jsons/valid_opcua.json") + spec = opcua.OpcuaSpec() + spec.load("tests/jsons/valid.json") message_structure = spec.get_code_name("INVALID", b"INVALID") self.assertEqual(message_structure, None) - class Test02OpcuaField(unittest.TestCase): """Test class for field crafting and access. Note that we don't test for the whole BOFField behavior, but From 282139183807551aa1a99eb4c935a6be7c86e626 Mon Sep 17 00:00:00 2001 From: Lex Date: Tue, 4 Aug 2020 17:45:42 +0200 Subject: [PATCH 15/24] Prepare for refactoring of common parts from knx and opcua --- bof/frame.py | 62 +++++++++++++++++++++++------------ bof/knx/knxframe.py | 29 ++++++----------- bof/knx/knxnet.json | 43 +++++++++++++++++++++---- bof/spec.py | 48 ++++++++++++++++++++++++++- examples/cemi.py | 68 --------------------------------------- tests/test_knx_frame.py | 8 ++--- tests/test_opcua_frame.py | 1 + 7 files changed, 141 insertions(+), 118 deletions(-) delete mode 100644 examples/cemi.py diff --git a/bof/frame.py b/bof/frame.py index 6441d51..83a4163 100644 --- a/bof/frame.py +++ b/bof/frame.py @@ -17,6 +17,12 @@ from .base import BOFProgrammingError, to_property, log from . import byte, spec +############################################################################### +# CONSTANTS # +############################################################################### + +PARENT = "parent" + ############################################################################### # Bit field representation within a field # ############################################################################### @@ -118,6 +124,7 @@ class BOFField(object): _name:str _size:int _value:bytes + _parent:object _is_length:bool _fixed_size:bool _fixed_value:bool @@ -125,21 +132,22 @@ class BOFField(object): _bitsizes:list def __init__(self, **kwargs): - self.name = kwargs["name"] if "name" in kwargs else "" - self._value = kwargs["value"] if "value" in kwargs else b'' - self._size = int(kwargs["size"]) if "size" in kwargs else max(1, byte.get_size(self._value)) - self._is_length = kwargs["is_length"] if "is_length" in kwargs else False - self._fixed_size = kwargs["fixed_size"] if "fixed_size" in kwargs else False - self._fixed_value = kwargs["fixed_value"] if "fixed_value" in kwargs else False + self.name = kwargs[spec.NAME] if spec.NAME in kwargs else "" + self._value = kwargs[spec.VALUE] if spec.VALUE in kwargs else b'' + self._size = int(kwargs[spec.SIZE]) if spec.SIZE in kwargs else max(1, byte.get_size(self._value)) + self._parent = kwargs[PARENT] if PARENT in kwargs else None + self._is_length = kwargs[spec.IS_LENGTH] if spec.IS_LENGTH in kwargs else False + self._fixed_size = kwargs[spec.F_SIZE] if spec.F_SIZE in kwargs else False + self._fixed_value = kwargs[spec.F_VALUE] if spec.F_VALUE in kwargs else False self._set_bitfields(**kwargs) # From now on, _update_value must be used to modify values within the code - if "optional" in kwargs and kwargs["optional"] and self._value == b'': + if spec.OPTIONAL in kwargs and kwargs[spec.OPTIONAL] and self._value == b'': self._size = 0 # We create the field byt don't use it. return - if "value" in kwargs and kwargs["value"] != b'': - self._update_value(kwargs["value"]) - elif "default" in kwargs: - self._update_value(kwargs["default"]) + if spec.VALUE in kwargs and kwargs[spec.VALUE] != b'': + self._update_value(kwargs[spec.VALUE]) + elif spec.DEFAULT in kwargs: + self._update_value(kwargs[spec.DEFAULT]) else: self._update_value(bytes(self._size)) @@ -153,10 +161,10 @@ def _set_bitfields(self, **kwargs): self._bitsizes = None if not spec.SEPARATOR in self._name: return - if "bitsizes" not in kwargs: + if spec.BITSIZES not in kwargs: raise BOFProgrammingError("Fields with bit fields shall have bitsizes ({0}).".format(self._name)) self._name = [x.strip() for x in self._name.split(spec.SEPARATOR)] # Now it's a table - self._bitsizes = [int(x) for x in kwargs["bitsizes"].split(spec.SEPARATOR)] + self._bitsizes = [int(x) for x in kwargs[spec.BITSIZES].split(spec.SEPARATOR)] if len(self._bitsizes) != len(self._name): raise BOFProgrammingError("Bitfield names do not match bitsizes ({0}).".format(self._name)) self._bitfields = {} @@ -262,8 +270,18 @@ def bitfield(self) -> dict: class BOFBlock(object): """A ``BOFBlock`` object represents a block (set of fields) within a - frame. It contains an ordered set of nested blocks and/or fields - (``BOFField``). + frame. It contains an ordered set of items. Items are nested blocks and/or + fields (``BOFField``). + + A block is usually built from a template which gives its structure. + Bytes can also be specified to "fill" this block stucture. If the bytes + values are consistent, the structure can also be determined directly from + them. If no structure is specified the block remains empty. + + Some block field value (typically a sub-block type) may depend on the + value of another field. In that case the keyword "depends:" is used to + associate the variable to its value, based on given parameters to another + field. Implementations should inherit this class for block management inside frames. @@ -273,6 +291,7 @@ class BOFBlock(object): :param parent: Parent frame, used when a field or a block depends on the value of a field previously written to the frame. :param content: List of blocks, fields or both. + :param spec: Specification storage class (inheriting from ``BOFSPec``). """ _name:str _content:list @@ -281,9 +300,12 @@ class BOFBlock(object): @classmethod def factory(cls, template) -> object: - """Class method to use when the object to create is not - necessarily a BOFBlock class. It should be instantiated - in protocol implementation classes. + """Class method to use when the object to create is not necessarily a + BOFBlock class. It should be instantiated in protocol implementation + classes as we need to instantiate protocol-specific block and field + classes and not BOFBlock and BOFField objects. + + This part may be replaced later. """ raise NotImplementedError("Factory should be instantiated in subclasses.") @@ -299,8 +321,8 @@ def __init__(self, **kwargs): values of fields to depend on to create a field (ex: message code). Defaults values are transmitted to children. """ - self.name = kwargs["name"] if "name" in kwargs else "" - self._parent = kwargs["parent"] if "parent" in kwargs else None + self.name = kwargs[spec.NAME] if spec.NAME in kwargs else "" + self._parent = kwargs[PARENT] if PARENT in kwargs else None self._content = [] def __bytes__(self): diff --git a/bof/knx/knxframe.py b/bof/knx/knxframe.py index 264640f..190e9dd 100644 --- a/bof/knx/knxframe.py +++ b/bof/knx/knxframe.py @@ -31,8 +31,7 @@ from ..base import BOFProgrammingError, to_property, log from ..frame import BOFFrame, BOFBlock, BOFField, BOFBitField -from ..spec import BOFSpec -from .. import byte +from .. import byte, spec ############################################################################### # KNX SPECIFICATION CONTENT # @@ -40,7 +39,7 @@ KNXSPECFILE = "knxnet.json" -class KnxSpec(BOFSpec): +class KnxSpec(spec.BOFSpec): """Singleton class for KnxSpec specification content usage. Inherits ``BOFSpec``. @@ -54,23 +53,15 @@ def __init__(self, filepath:str=None): filepath = path.join(path.dirname(path.realpath(__file__)), KNXSPECFILE) super().__init__(filepath) - #-------------------------------------------------------------------------# - # Public # - #-------------------------------------------------------------------------# - - def get_block_template(self, name:str) -> list: - """Returns a template associated to a body, as a list, or None.""" - return self._get_dict_value(self.blocks, name) if name else None - - def get_code_name(self, dict_key:str, identifier) -> str: - dict_key = self._get_dict_key(self.codes, dict_key) + def get_code_name(self, code:str, identifier) -> str: + code = self._get_dict_key(self.codes, code) if isinstance(identifier, bytes): - for key in self.codes[dict_key]: + for key in self.codes[code]: if identifier == bytes.fromhex(key): - return self.codes[dict_key][key] + return self.codes[code][key] if isinstance(identifier, str): identifier = to_property(identifier) - for service in self.codes["service identifier"].values(): + for service in self.codes[code].values(): if identifier == to_property(service): return service return None @@ -156,7 +147,7 @@ def factory(cls, template, **kwargs) -> object: with format {"field name": b"value"} :param value: Content of block or field to set. """ - if "type" in template and template["type"] == "field": + if spec.TYPE in template and template[spec.TYPE] == spec.FIELD: value = b'' if "defaults" in kwargs and template["name"] in kwargs["defaults"]: value = kwargs["defaults"][template["name"]] @@ -173,7 +164,7 @@ def __init__(self, **kwargs): self._spec = KnxSpec() super().__init__(**kwargs) # Without a type, the block remains empty - if not "type" in kwargs or kwargs["type"] == "block": + if not spec.TYPE in kwargs or kwargs[spec.TYPE] == spec.BLOCK: return # Now we extract the final type of block from the arguments value = kwargs["value"] if "value" in kwargs else None @@ -230,7 +221,7 @@ class KnxFrame(BOFFrame): # {Argument name: field name} "type": "service identifier", "cemi": "message code", - "connection": "connection type code" + "connection": "cri connection type code" } def __init__(self, **kwargs): diff --git a/bof/knx/knxnet.json b/bof/knx/knxnet.json index cb29c65..f8637ed 100644 --- a/bof/knx/knxnet.json +++ b/bof/knx/knxnet.json @@ -32,13 +32,13 @@ "CONNECT REQUEST": [ {"name": "control endpoint", "type": "HPAI"}, {"name": "data endpoint", "type": "HPAI"}, - {"name": "connection request information", "type": "CRI_CRD"} + {"name": "connection request information", "type": "CRI"} ], "CONNECT RESPONSE": [ {"name": "communication channel id", "type": "field", "size": 1}, {"name": "status", "type": "field", "size": 1}, {"name": "data endpoint", "type": "HPAI"}, - {"name": "connection response data block", "type": "CRI_CRD"} + {"name": "connection response data block", "type": "CRD"} ], "CONNECTIONSTATE REQUEST": [ {"name": "communication channel id", "type": "field", "size": 1}, @@ -98,10 +98,15 @@ {"name": "id", "type": "field", "size": 1}, {"name": "version", "type": "field", "size": 1} ], - "CRI_CRD": [ + "CRI": [ {"name": "structure length", "type": "field", "size": 1, "is_length": true}, - {"name": "connection type code", "type": "field", "size": 1, "default": "03"}, - {"name": "connection data", "type": "depends:connection type code"} + {"name": "cri connection type code", "type": "field", "size": 1, "default": "03"}, + {"name": "connection data", "type": "depends:cri connection type code"} + ], + "CRD": [ + {"name": "structure length", "type": "field", "size": 1, "is_length": true}, + {"name": "crd connection type code", "type": "field", "size": 1, "default": "03"}, + {"name": "connection data", "type": "depends:crd connection type code"} ], "DEVICE_MANAGEMENT_CONNECTION": [ {"name": "ip address", "type": "field", "size": 4, "optional": true}, @@ -110,12 +115,26 @@ {"name": "port", "type": "field", "size": 2, "optional": true} ], "TUNNELING_CONNECTION": [ + {"name": "knx layer", "type": "field", "size": 1, "default": "02"}, + {"name": "reserved", "type": "field", "size": 1} + ], + "CRD TUNNELING_CONNECTION": [ {"name": "knx address", "type": "field", "size": 2} ], "CEMI": [ {"name": "message code", "type": "field", "size": 1}, {"name": "cemi data", "type": "depends:message code"} ], + "L_cEMI": [ + {"name": "additional info length", "type": "field", "size": 1, "default": "00"}, + {"name": "additional information", "type": "field", "size": "depends:additional info length"}, + {"name": "frame type, empty, repeat, system broadcast, priority, ack request, confirm", "type": "field", "size": 1, "bitsizes": "1, 1, 1, 1, 2, 1, 1"}, + {"name": "destination address, hop count, extended frame format", "type": "field", "size": 1, "bitsizes": "1, 3, 4"}, + {"name": "source address", "type": "field", "size": 2}, + {"name": "destination address", "type": "field", "size": 2}, + {"name": "information length", "type": "field", "size": 1, "default": "01"}, + {"name": "tpci apci", "type": "field", "size": "depends: information length"} + ], "DP_cEMI": [ {"name": "object type", "type": "field", "size": 2}, {"name": "object instance", "type": "field", "size": 1}, @@ -123,6 +142,12 @@ {"name": "number of elements, start index", "type": "field", "size": 2, "bitsizes": "4, 12"}, {"name": "data", "type": "field", "size": 0} ], + "L_Data.req": [ + {"name": "L_Data.req", "type": "L_cEMI"} + ], + "L_Data.con": [ + {"name": "L_Data.con", "type": "L_cEMI"} + ], "PropRead.req": [ {"name": "PropRead.req", "type": "DP_cEMI"} ], @@ -153,14 +178,20 @@ "0311": "CONFIGURATION ACK" }, "message code": { + "11": "L_Data.req", + "2e": "L_Data.con", "FC": "PropRead.req", "FB": "PropRead.con", "F6": "PropWrite.req", "F5": "PropWrite.con" }, - "connection type code": { + "cri connection type code": { "03": "DEVICE MANAGEMENT CONNECTION", "04": "TUNNELING CONNECTION" + }, + "crd connection type code": { + "03": "DEVICE MANAGEMENT CONNECTION", + "04": "CRD TUNNELING CONNECTION" } }, "object types": { diff --git a/bof/spec.py b/bof/spec.py index 95eea82..8d36312 100644 --- a/bof/spec.py +++ b/bof/spec.py @@ -36,7 +36,24 @@ # Global structure # #-----------------------------------------------------------------------------# +# GENERAL SYNTAX SEPARATOR = "," +DEPENDS = "depends:" +# ITEMS (FIELDS AND BLOCKS) +NAME = "name" +TYPE = "type" +VALUE = "value" +SIZE = "size" +OPTIONAL = "optional" +DEFAULT = "default" +# FIELDS +FIELD = "field" +IS_LENGTH = "is_length" +F_SIZE = "fixed_size" +F_VALUE = "fixed_value" +BITSIZES = "bitsizes" +# BLOCKS +BLOCK = "block" ############################################################################### # JSON file management functions # @@ -84,7 +101,7 @@ def __init__(self, filepath:str=None): self.__is_init = True #-------------------------------------------------------------------------# - # Public # + # Public JSON file management methods # #-------------------------------------------------------------------------# def load(self, filepath): @@ -119,6 +136,35 @@ def clear(self): for key in attributes: delattr(self, key) + #-------------------------------------------------------------------------# + # Public getters # + #-------------------------------------------------------------------------# + + def get_item_template(self, block_name:str, item_name:str) -> dict: + """Returns an item template (dict of values) from a `block_name` and + a `field_name`. + + Note that an item template can represent either a field or a block, + depending on the "type" key of the item. + + :param block_name: Name of the block containing the item. + :param item_name: Name of the item to look for in the block. + :returns: Item template associated to block_name and item_name + """ + if block_name in self.blocks: + for item in self.blocks[block_name]: + if item[NAME] == item_name: + return item + return None + + def get_block_template(self, block_name:str) -> list: + """Returns a block template (list of item templates) from a block name. + + :param block_name: Name of the block we want the template from. + :returns: Block template associated with the specifified block_name. + """ + return self._get_dict_value(self.blocks, block_name) if block_name else None + #-------------------------------------------------------------------------# # Internals # #-------------------------------------------------------------------------# diff --git a/examples/cemi.py b/examples/cemi.py deleted file mode 100644 index afc3d7b..0000000 --- a/examples/cemi.py +++ /dev/null @@ -1,68 +0,0 @@ -from sys import path, argv -path.append('../') - -from bof import knx, BOFNetworkError, byte - -def connect_request(knxnet, connection_type): - ip, port = knxnet.source - connectreq = knx.KnxFrame(type="CONNECT REQUEST") - connectreq.body.connection_request_information.connection_type_code.value = knx.KnxSpec().get_code_id("connection type code", connection_type) - connectreq.body.control_endpoint.ip_address.value = ip - connectreq.body.control_endpoint.port.value = port - connectreq.body.data_endpoint.ip_address.value = ip - connectreq.body.data_endpoint.port.value = port - if connection_type == "Tunneling Connection": - connectreq.body.connection_request_information.append(knx.KnxField(name="link layer", size=1, value=b"\x02")) - connectreq.body.connection_request_information.append(knx.KnxField(name="reserved", size=1, value=b"\x00")) - connectresp = knxnet.send_receive(connectreq) - knxnet.channel = connectresp.body.communication_channel_id.value - return connectresp - -def disconnect_request(knxnet): - discoreq = knx.KnxFrame(type="DISCONNECT REQUEST") - discoreq.body.communication_channel_id = knxnet.channel - discoreq.body.control_endpoint.ip_address.value = byte.from_ipv4(knxnet.source[0]) - discoreq.body.control_endpoint.port.value = byte.from_int(knxnet.source[1]) - knxnet.send_receive(discoreq) - -def read_property(knxnet, sequence_counter, object_type, property_id): - request = knx.KnxFrame(type="CONFIGURATION REQUEST", cemi="PropRead.req") - request.body.communication_channel_id.value = knxnet.channel - request.body.sequence_counter.value = sequence_counter - propread = request.body.cemi.cemi_data.propread_req - propread.number_of_elements.value = 1 - propread.object_type.value = knxspecs.object_types[object_type] - propread.object_instance.value = 1 - propread.property_id.value = knxspecs.properties[object_type][property_id] - try: - response = knxnet.send_receive(request) # ACK - while (1): - response = knxnet.receive() # PropRead.con - if response.sid == "CONFIGURATION REQUEST": - # We tell the boiboite we received it - ack = knx.KnxFrame(type="CONFIGURATION ACK") - ack.body.communication_channel_id.value = knxnet.channel - ack.body.sequence_counter.value = sequence_counter - knxnet.send(ack) - return response - except BOFNetworkError: - pass #Timeout - -if len(argv) < 2: - print("Usage: python {0} IP_ADDRESS".format(argv[0])) - quit() - -knxspecs = knx.KnxSpec() -knxnet = knx.KnxNet() -knxnet.connect(argv[1], 3671) - -# Gather device information -connectresp = connect_request(knxnet, "Device Management Connection") -read_property(knxnet, 0, "IP PARAMETER OBJECTS", "PID_ADDITIONAL_INDIVIDUAL_ADDRESSES") -disconnect_request(knxnet) - -# Establish tunneling connection to read and write objects -connectresp = connect_request(knxnet, "Tunneling Connection") -print("Device individual address: {0}".format(connectresp.body.connection_response_data_block.connection_data.knx_address.value)) -# TODO -disconnect_request(knxnet) diff --git a/tests/test_knx_frame.py b/tests/test_knx_frame.py index 84f2c3c..341c567 100644 --- a/tests/test_knx_frame.py +++ b/tests/test_knx_frame.py @@ -246,8 +246,8 @@ def test_01_knx_parse_descrresp(self): self.assertEqual(bytes(datagram.header.service_identifier), b"\x02\x04") def test_02_knx_parse_connectresp(self): connectreq = knx.KnxFrame(type="CONNECT_REQUEST") - connectreq.body.connection_request_information.connection_type_code.value = \ - knx.KnxSpec().get_code_id("connection type code", "Device Management Connection") + connectreq.body.connection_request_information.cri_connection_type_code.value = \ + knx.KnxSpec().get_code_id("cri connection type code", "Device Management Connection") self.connection.send(connectreq) connectresp = self.connection.receive() channel = connectresp.body.communication_channel_id.value @@ -280,8 +280,8 @@ def test_03_knx_cemi_bitfields_parsing(self): knxnet.connect(BOIBOITE, 3671) # ConnectReq connectreq = knx.KnxFrame(type="CONNECT REQUEST") - connectreq.body.connection_request_information.connection_type_code.value = \ - knx.KnxSpec().get_code_id("connection type code", "Device Management Connection") + connectreq.body.connection_request_information.cri_connection_type_code.value = \ + knx.KnxSpec().get_code_id("cri connection type code", "Device Management Connection") connectreq.body.control_endpoint.ip_address.value = byte.from_ipv4(knxnet.source[0]) connectreq.body.control_endpoint.port.value = byte.from_int(knxnet.source[1]) connectreq.body.data_endpoint.ip_address.value = byte.from_ipv4(knxnet.source[0]) diff --git a/tests/test_opcua_frame.py b/tests/test_opcua_frame.py index 79a9563..3ad9de8 100644 --- a/tests/test_opcua_frame.py +++ b/tests/test_opcua_frame.py @@ -6,6 +6,7 @@ import unittest from bof import opcua, BOFLibraryError, BOFProgrammingError +@unittest.skip("Rework JSON call") class Test01OpcuaSpec(unittest.TestCase): """Test class for specification class building from JSON file. TODO: Some tests are generic to BOFSpec and could be moved. From 0a28cc98a1f7bbd9f56a1c364935ae5b6873e7cc Mon Sep 17 00:00:00 2001 From: Lex Date: Wed, 5 Aug 2020 11:34:38 +0200 Subject: [PATCH 16/24] Refactoring of BOFBlock using common stuff from KNX and OPCUA implementation classes --- bof/frame.py | 108 +++++++++++++++++++++++++++++++++++----- bof/knx/knxframe.py | 51 +++++-------------- bof/opcua/opcuaframe.py | 84 ++++++++----------------------- 3 files changed, 129 insertions(+), 114 deletions(-) diff --git a/bof/frame.py b/bof/frame.py index 83a4163..4bcc5fc 100644 --- a/bof/frame.py +++ b/bof/frame.py @@ -22,6 +22,8 @@ ############################################################################### PARENT = "parent" +VALUE = "value" +USER_VALUES = "user_values" ############################################################################### # Bit field representation within a field # @@ -310,20 +312,41 @@ def factory(cls, template) -> object: raise NotImplementedError("Factory should be instantiated in subclasses.") def __init__(self, **kwargs): - """Initialize a block according to a set or arguments (template). + """Initialize a block according to a set or arguments from an item + template (dictionary inside a spec JSON file) and directly from kwargs + given to the constructor when creating the block object instance. - A template usually contains the following information and has the - following format in a protocol's specification file: + Requires a specification file, therefore this constructor cannot be + used directly and must be called from a subclass init method, such as:: - {"name": "control endpoint", "type": "HPAI"}, + self._spec = KnxSpec() + super().__init__(**kwargs) - :param defaults: Dictionary for optional keyword arguments to force - values of fields to depend on to create a field (ex: message code). - Defaults values are transmitted to children. + Calls the public method ``build()`` to create the structure and fill + items. Refer to its docstrings to know what type of arguments is + expected here. + + Optional keyword arguments: + + :param name: The name of the block. If empty and the block has a type, + block name == block type + :param parent: The parent block (``BOFBlock`` instance), if any. """ + # Check that the specification object has been defined in subclass + # before calling this constructor. + if not hasattr(self, "_spec") or not isinstance(self._spec, spec.BOFSpec): + raise BOFProgrammingError("BOFBlock cannot be instantiated directly " \ + "and requires previous initialization of a BOFSpec object in the " \ + "subclass' constructor.") + # Basic block information self.name = kwargs[spec.NAME] if spec.NAME in kwargs else "" self._parent = kwargs[PARENT] if PARENT in kwargs else None self._content = [] + # Create and fill the block + self.build(**kwargs) + # If we still don't have a name, we try to set one + if not len(self.name) and spec.TYPE in kwargs: + self.name = kwargs[spec.TYPE] def __bytes__(self): return b''.join(bytes(item) for item in self._content) @@ -344,6 +367,56 @@ def __iter__(self): # Public # #-------------------------------------------------------------------------# + def build(self, **kwargs): + """Create and fill the KnxBlock from an item template extracted from + the JSON file and additional arguments if any. + + A template usually contains the following information and has the + following format in a protocol's specification file: + + {"name": "control endpoint", "type": "HPAI"}, + + Mandatory keyword arguments: + + :param type: Type of the block. If no type is set or type is block, + we don't know how to build the structure, we stop here. + + Optional keyword arguments: + + :param value: Byte array (usually a received frame) to fill the block + :param user_values: Dictionary for optional keyword arguments to force + values of fields to depend on to create a field + (ex: message code). Transmitted to children. + + :raises BOFProgrammingError: If specified type was not found in the + JSON spec file's blocks list or if the + format found is invalid. + """ + if not spec.TYPE in kwargs or kwargs[spec.TYPE] == spec.BLOCK: + return + user_values = kwargs[USER_VALUES] if USER_VALUES in kwargs else {} + value = kwargs[VALUE] if VALUE in kwargs else None + # If values rely on previous content, replace it + for key in kwargs: + if isinstance(kwargs[key], str) and kwargs[key].startswith(spec.DEPENDS): + dependency = kwargs[key].split(spec.DEPENDS)[1] + kwargs[key] = self._get_depends_block(dependency, user_values) + # Retrieve the template in the JSON file and check it + block_template = self._spec.get_block_template(kwargs[spec.TYPE]) + if not block_template: + raise BOFProgrammingError("Unknown block type ({0})".format(kwargs[spec.TYPE])) + if not isinstance(block_template, list): + raise BOFProgrammingError("Invalid block format ({0})".format(kwargs[spec.TYPE])) + # Create block and fill them (if value) one by one + for item_template in block_template: + item = self.factory(item_template, value=value, user_values=user_values, parent=self) + self.append(item) + # If value, we extract part of it to fill the item + if value: + if len(item) >= len(value): + break + value = value[len(item):] + def append(self, content) -> None: """Appends a block, a field or a list of blocks and/fields to current block's content. Adds the name of the block to the list @@ -434,25 +507,34 @@ def _add_property(self, name, pointer:object) -> None: elif len(name) > 0: setattr(self, to_property(name), pointer) - def _get_depends_block(self, field:str, defaults:dict=None): + def _get_depends_block(self, field:str, user_values:dict=None): """If the format of a block depends on the value of a field set previously, we look for it and choose the appropriate format. The closest field with such name is used. :param name: Name of the field to look for and extract value. - :raises BOFProgrammingError: If specified field was not found. + :raises BOFProgrammingError: If specified field was not found or no + association was found. """ field = to_property(field) + # First look in user-defined parameter values + if user_values: + for key in user_values: + if field == to_property(key): + block = self._spec.get_code_name(key, user_values[key]) + if block: + return block + # Then look in previously-set fields (starting by the closest ones) if self._parent: field_list = list(self._parent) field_list.reverse() for frame_field in field_list: if field == to_property(frame_field.name): block = self._spec.get_code_name(frame_field.name, frame_field.value) - return block - raise BOFProgrammingError("Field not found ({0}).".format(field)) - else: - return None + if block: + return block + raise BOFProgrammingError("Association not found for field {0}".format(field)) + #-------------------------------------------------------------------------# # Properties # diff --git a/bof/knx/knxframe.py b/bof/knx/knxframe.py index 190e9dd..7d4c437 100644 --- a/bof/knx/knxframe.py +++ b/bof/knx/knxframe.py @@ -30,7 +30,7 @@ from textwrap import indent from ..base import BOFProgrammingError, to_property, log -from ..frame import BOFFrame, BOFBlock, BOFField, BOFBitField +from ..frame import BOFFrame, BOFBlock, BOFField, USER_VALUES, VALUE from .. import byte, spec ############################################################################### @@ -143,16 +143,16 @@ def factory(cls, template, **kwargs) -> object: Keyword arguments: - :param defaults: Default values to assign a field as a dictionary - with format {"field name": b"value"} + :param user_values: Default values to assign a field as a dictionary + with format {"field name": b"value"} :param value: Content of block or field to set. """ if spec.TYPE in template and template[spec.TYPE] == spec.FIELD: value = b'' - if "defaults" in kwargs and template["name"] in kwargs["defaults"]: - value = kwargs["defaults"][template["name"]] - elif "value" in kwargs and kwargs["value"]: - value = kwargs["value"][:template["size"]] + if USER_VALUES in kwargs and template[spec.NAME] in kwargs[USER_VALUES]: + value = kwargs[USER_VALUES][template[spec.NAME]] + elif VALUE in kwargs and kwargs[VALUE]: + value = kwargs[VALUE][:template[spec.SIZE]] return KnxField(**template, value=value) return cls(**template, **kwargs) @@ -163,33 +163,6 @@ def __init__(self, **kwargs): """ self._spec = KnxSpec() super().__init__(**kwargs) - # Without a type, the block remains empty - if not spec.TYPE in kwargs or kwargs[spec.TYPE] == spec.BLOCK: - return - # Now we extract the final type of block from the arguments - value = kwargs["value"] if "value" in kwargs else None - defaults = kwargs["defaults"] if "defaults" in kwargs else {} - block_type = kwargs["type"] - if block_type.startswith("depends:"): - field_name = to_property(block_type.split(":")[1]) - block_type = self._get_depends_block(field_name, defaults) - if not block_type: - raise BOFProgrammingError("Association not found for field {0}".format(field_name)) - # We extract the block's content according to its type - template = self._spec.get_block_template(block_type) - if not template: - raise BOFProgrammingError("Unknown block type ({0})".format(block_type)) - # And we fill the block according to its content - template = [template] if not isinstance(template, list) else template - for item in template: - new_item = self.factory(item, value=value, - defaults=defaults, parent=self) - self.append(new_item) - # Update value - if value: - if len(new_item) >= len(value): - break - value = value[len(new_item):] #-----------------------------------------------------------------------------# # KNX frames / datagram representation # @@ -217,7 +190,7 @@ class KnxFrame(BOFFrame): **KNX Standard v2.1 03_08_02** """ - __defaults = { + __user_values = { # {Argument name: field name} "type": "service identifier", "cemi": "message code", @@ -254,14 +227,14 @@ def __init__(self, **kwargs): super().__init__() # We store some values before starting building the frame value = kwargs["bytes"] if "bytes" in kwargs else None - defaults = {} - for arg, code in self.__defaults.items(): + user_values = {} + for arg, code in self.__user_values.items(): if arg in kwargs: - defaults[code] = spec.get_code_id(code, kwargs[arg]) + user_values[code] = spec.get_code_id(code, kwargs[arg]) # Now we can start for block in spec.frame: # Create block - knxblock = KnxBlock(value=value, defaults=defaults, parent=self, **block) + knxblock = KnxBlock(value=value, user_values=user_values, parent=self, **block) self.append(block["name"], knxblock) # If a value is used to fill the blocks, update it if value: diff --git a/bof/opcua/opcuaframe.py b/bof/opcua/opcuaframe.py index 2a84b11..c79d8ba 100644 --- a/bof/opcua/opcuaframe.py +++ b/bof/opcua/opcuaframe.py @@ -14,9 +14,8 @@ from os import path from ..base import BOFProgrammingError, to_property, log -from ..frame import BOFFrame, BOFBlock, BOFField -from ..spec import BOFSpec -from .. import byte +from ..frame import BOFFrame, BOFBlock, BOFField, USER_VALUES, VALUE +from .. import byte, spec ############################################################################### # OPCUA SPECIFICATION CONTENT # @@ -24,7 +23,7 @@ OPCUASPECFILE = "opcua.json" -class OpcuaSpec(BOFSpec): +class OpcuaSpec(spec.BOFSpec): """Singleton class for OPC UA specification content usage. Inherits ``BOFSpec``, see `bof/frame.py`. @@ -194,16 +193,16 @@ def factory(cls, item_template:dict, **kwargs) -> object: Keyword arguments: - :param defaults: Defaults values to assign a field as dictionnary. + :param user_values: Default values to assign a field as dictionnary. :param value: Bytes value to fill the item (block or field) with. """ # case where item template represents a field (non-recursive) - if "type" in item_template and item_template["type"] == "field": + if spec.TYPE in item_template and item_template[spec.TYPE] == spec.FIELD: value = b'' - if "defaults" in kwargs and item_template["name"] in kwargs["defaults"]: - value = kwargs["defaults"][item_template["name"]] - elif "value" in kwargs and kwargs["value"]: - value = kwargs["value"][:item_template["size"]] + if USER_VALUES in kwargs and item_template[spec.NAME] in kwargs[USER_VALUES]: + value = kwargs[USER_VALUES][item_template[spec.NAME]] + elif VALUE in kwargs and kwargs[VALUE]: + value = kwargs[VALUE][:item_template[spec.SIZE]] return OpcuaField(**item_template, value=value) # case where item template represents a sub-block (nested/recursive block) else: @@ -216,61 +215,22 @@ def __init__(self, **kwargs): :param type: a string specifying block type (as found in json specifications) to construct the block on. - :param defaults: defaults values to assign a field as dictionnary + :param user_values: default values to assign a field as dictionnary (can therefore be used to construct blocks with dependencies if not found in raw bytes, see example above) :param value: bytes value to fill the block with (can create dependencies on its own, see example - above). If defaults parameter is found it overcomes + above). If user_values parameter is found it overcomes the value passed as bytes. See example in class docstring to understand dependency creation - either with defaults or with value parameter. + either with user_values or value parameter. """ self._spec = OpcuaSpec() super().__init__(**kwargs) - # we gather args values and set some default values first - defaults = kwargs["defaults"] if "defaults" in kwargs else {} - value = kwargs["value"] if "value" in kwargs else {} - self._name = kwargs["name"] if "name" in kwargs else "" - - # we gather the block type from args, if not found then returns an - # empty block - if "type" in kwargs and kwargs["type"] != "block": - block_type = kwargs["type"] - else: - log("No type or item_template_block specified, creating empty block") - return - - # if a dependency is found in block type - # looks for needed information in default arg - if block_type.startswith("depends:"): - dependency = block_type.split(":")[1] - block_type = self._get_depends_block(dependency) - if not block_type: - raise BOFProgrammingError("No valid association found in parents for dependency '{0}'.".format(dependency)) - log("Creating OpcuaBlock of type '{0}' from dependency '{1}'.".format(block_type, dependency)) - else: - log("Creating OpcuaBlock of type '{0}'.".format(block_type)) - - # we extract the block template according to its type - block_template = self._spec.get_block_template(block_type) - if not block_template: - raise BOFProgrammingError("Block type '{0}' not found in specifications.".format(block_type)) - - for item_template in block_template: - new_item = self.factory(item_template, value=value, defaults=defaults, parent=self) - self.append(new_item) - - # Cut value to fill byte by byte - if value: - if len(new_item) >= len(value): - break - value = value[len(new_item):] - #-----------------------------------------------------------------------------# # OPC UA frames representation # #-----------------------------------------------------------------------------# @@ -284,7 +244,7 @@ class OpcuaFrame(BOFFrame): :Byte array: Build the object from a raw byte array, typically used when receiving incoming connection. In this case block dependencies are identified automatically. - :Keyword arguments: Uses keyword described in __defaults to fill frame + :Keyword arguments: Uses keyword described in __user_values to fill frame fields. Usage example:: @@ -298,7 +258,7 @@ class OpcuaFrame(BOFFrame): frame = opcua.OpcuaFrame(type="HEL") """ - __defaults = { + __user_values = { # {Argument name: field name} "type": "message_type", } @@ -308,23 +268,23 @@ def __init__(self, **kwargs): keyword arguments : :param byte: raw byte array used to build a frame. - :defaults arguments: every element of __default specifies arguments - that can be passed in order to set fields values - at frame creation. + :param user_values: every element of __default specifies arguments + that can be passed in order to set fields values + at frame creation. """ spec = OpcuaSpec() super().__init__() # We store some values before starting building the frame value = kwargs["bytes"] if "bytes" in kwargs else None - defaults = {} - for arg, code in self.__defaults.items(): + user_values = {} + for arg, code in self.__user_values.items(): if arg in kwargs: - defaults[code] = str.encode(kwargs[arg]) + user_values[code] = str.encode(kwargs[arg]) # Now we can start for block in spec.frame: # Create block - opcuablock = OpcuaBlock(value=value, defaults=defaults, parent=self, **block) + opcuablock = OpcuaBlock(value=value, user_values=user_values, parent=self, **block) self.append(block["name"], opcuablock) # If a value is used to fill the blocks, update it if value: @@ -358,4 +318,4 @@ def header(self): @property def body(self): self.update() - return self._blocks["body"] \ No newline at end of file + return self._blocks["body"] From 08c71fbe81fbc682a74fccc908f53a5e6db675a8 Mon Sep 17 00:00:00 2001 From: Lex Date: Wed, 5 Aug 2020 14:50:02 +0200 Subject: [PATCH 17/24] Common frame content moved to BOFFrame, req some docstring cleaning in implementations --- bof/frame.py | 64 ++++++++++++++++++++++++++++++++++++++++- bof/knx/knxframe.py | 48 ++++++++++++------------------- bof/opcua/opcuaframe.py | 34 ++++++---------------- bof/spec.py | 3 ++ examples/cemi_fuzzer.py | 5 ++-- 5 files changed, 95 insertions(+), 59 deletions(-) diff --git a/bof/frame.py b/bof/frame.py index 4bcc5fc..214f544 100644 --- a/bof/frame.py +++ b/bof/frame.py @@ -22,6 +22,7 @@ ############################################################################### PARENT = "parent" +BYTES = "bytes" VALUE = "value" USER_VALUES = "user_values" @@ -583,16 +584,77 @@ class BOFFrame(object): - The frame contains a set of blocks - The order of blocks is defined, blocks are named. + In this class, a frame is built according to a special part of a JSON + specification file, which has the following name and format:: + + "frame": [ + {"name": "header", "type": "HEADER"}, + {"name": "body", "type": "depends:message_type"} + ] + + Attributes: + :param blocks: A dictionary containing blocks. + :param spec: The specification class as a ``BOFSpec`` object. Should be + instantiated in a subclass as BOFFrame should never be + used directly. + :param user_args: A dictionary containing the name of an argument that a end + user can supply when creating the frame object instance, + and the name of the corresponding field in the frame + accçording to the JSON spec file. For instance: + ``"type": "message_type"`` states that the user can create + the frame object with ``OpcuaFrame(type="HEL") and that + the field value to look for or to fill is ``message_type`` .. warning: We rely on Python 3.6+'s ordering by insertion. If you use an older implementation of Python, blocks may not come in the right order (and I don't think BOF would work anyway). """ _blocks:dict + _spec:object + _user_args = { + # {Argument name: field name} + } + + def __init__(self, block_class:object, **kwargs): + """Create the frame according to the category "frame" in a JSON + specification file. + + Requires a specification file, as well as the type of block class it + has to create, therefore this constructor cannot be used directly and + must be called from a subclass init method, such as:: + + self._spec = KnxSpec() + super().__init__(KnxBlock, **kwargs) - def __init__(self): + :param block_class: Type of the class to create. Must inherit from + ``BOFBlock`.` + """ + # Check that the specification object has been defined in subclass + # before calling this constructor. + if not hasattr(self, "_spec") or not isinstance(self._spec, spec.BOFSpec): + raise BOFProgrammingError("BOFFrame cannot be instantiated directly " \ + "and requires previous initialization of a BOFSpec object in the " \ + "subclass' constructor.") + if not block_class or not isinstance(block_class(), BOFBlock): + raise BOFProgrammingError("BOFFrame expects a BOFBlock class type " \ + "from a protocol implementation as first argument.") self._blocks = {} + # Retrieve actual or default values to use to build the frame + user_values = {} + for arg, code in self._user_args.items(): + if arg in kwargs: + user_values[code] = self._spec.get_code_id(code, kwargs[arg]) + value = kwargs[BYTES] if BYTES in kwargs else None + # Now build the frame according to what the spec says. + for block_template in self._spec.frame: + block = block_class(value=value, user_values=user_values, + parent=self, **block_template) + self.append(block_template[spec.NAME], block) + if value: + if len(self._blocks[block_template[spec.NAME]]) >= len(value): + break + value = value[len(self._blocks[block_template[spec.NAME]]):] def __bytes__(self): self.update() diff --git a/bof/knx/knxframe.py b/bof/knx/knxframe.py index 7d4c437..2a7abb9 100644 --- a/bof/knx/knxframe.py +++ b/bof/knx/knxframe.py @@ -34,11 +34,18 @@ from .. import byte, spec ############################################################################### -# KNX SPECIFICATION CONTENT # +# KNX-related constants # ############################################################################### KNXSPECFILE = "knxnet.json" +TOTAL_LENGTH = "total_length" + +############################################################################### +# KNX SPECIFICATION CONTENT # +############################################################################### + + class KnxSpec(spec.BOFSpec): """Singleton class for KnxSpec specification content usage. Inherits ``BOFSpec``. @@ -190,7 +197,7 @@ class KnxFrame(BOFFrame): **KNX Standard v2.1 03_08_02** """ - __user_values = { + _user_args = { # {Argument name: field name} "type": "service identifier", "cemi": "message code", @@ -223,25 +230,8 @@ def __init__(self, **kwargs): :param *: Other params corresponding to default values can be given. The param name must be the name of the field to fill. """ - spec = KnxSpec() - super().__init__() - # We store some values before starting building the frame - value = kwargs["bytes"] if "bytes" in kwargs else None - user_values = {} - for arg, code in self.__user_values.items(): - if arg in kwargs: - user_values[code] = spec.get_code_id(code, kwargs[arg]) - # Now we can start - for block in spec.frame: - # Create block - knxblock = KnxBlock(value=value, user_values=user_values, parent=self, **block) - self.append(block["name"], knxblock) - # If a value is used to fill the blocks, update it - if value: - if len(self._blocks[block["name"]]) >= len(value): - break - value = value[len(self._blocks[block["name"]]):] - # Update total frame length in header + self._spec = KnxSpec() + super().__init__(KnxBlock, **kwargs) self.update() #-------------------------------------------------------------------------# @@ -255,9 +245,9 @@ def update(self): field in header, which requires an additional operation. """ super().update() - if "total_length" in self._blocks["header"].attributes: + if TOTAL_LENGTH in self._blocks[spec.HEADER].attributes: total = sum([len(block) for block in self._blocks.values()]) - self._blocks["header"].total_length._update_value(byte.from_int(total)) + self._blocks[spec.HEADER].total_length._update_value(byte.from_int(total)) #-------------------------------------------------------------------------# # Properties # @@ -266,22 +256,22 @@ def update(self): @property def header(self): self.update() - return self._blocks["header"] + return self._blocks[spec.HEADER] @property def body(self): self.update() - return self._blocks["body"] + return self._blocks[spec.BODY] @property def sid(self) -> str: """Return the name associated to the frame's service identifier, or empty string if it is not set. """ - sid = KnxSpec().get_code_name("service identifier", - self._blocks["header"].service_identifier.value) - return sid if sid else str(self._blocks["header"].service_identifier.value) + sid = self._spec.get_code_name("service identifier", + self._blocks[spec.HEADER].service_identifier.value) + return sid if sid else str(self._blocks[spec.HEADER].service_identifier.value) @property def cemi(self) -> str: """Return the type of cemi, if any.""" - KnxSpec().get_cemi_name(self._blocks["body"].cemi.message_code) + self._spec.get_cemi_name(self._blocks[spec.BODY].cemi.message_code) diff --git a/bof/opcua/opcuaframe.py b/bof/opcua/opcuaframe.py index c79d8ba..4a17814 100644 --- a/bof/opcua/opcuaframe.py +++ b/bof/opcua/opcuaframe.py @@ -236,7 +236,7 @@ def __init__(self, **kwargs): #-----------------------------------------------------------------------------# class OpcuaFrame(BOFFrame): - """Object representation of an OPC UA frame, created from the tempalte + """Object representation of an OPC UA frame, created from the template in `opcua.json`. Uses various initialization methods to create a frame : @@ -258,7 +258,7 @@ class OpcuaFrame(BOFFrame): frame = opcua.OpcuaFrame(type="HEL") """ - __user_values = { + _user_args = { # {Argument name: field name} "type": "message_type", } @@ -272,26 +272,8 @@ def __init__(self, **kwargs): that can be passed in order to set fields values at frame creation. """ - spec = OpcuaSpec() - super().__init__() - - # We store some values before starting building the frame - value = kwargs["bytes"] if "bytes" in kwargs else None - user_values = {} - for arg, code in self.__user_values.items(): - if arg in kwargs: - user_values[code] = str.encode(kwargs[arg]) - # Now we can start - for block in spec.frame: - # Create block - opcuablock = OpcuaBlock(value=value, user_values=user_values, parent=self, **block) - self.append(block["name"], opcuablock) - # If a value is used to fill the blocks, update it - if value: - if len(self._blocks[block["name"]]) >= len(value): - break - value = value[len(self._blocks[block["name"]]):] - # Update total frame length in header + self._spec = OpcuaSpec() + super().__init__(OpcuaBlock, **kwargs) self.update() #-------------------------------------------------------------------------# @@ -303,9 +285,9 @@ def update(self): frame length. """ #super().update() - if "message_size" in self._blocks["header"].attributes: + if "message_size" in self._blocks[spec.HEADER].attributes: total = sum([len(block) for block in self._blocks.values()]) - self._blocks["header"].message_size._update_value(byte.from_int(total)) + self._blocks[spec.HEADER].message_size._update_value(byte.from_int(total)) #-------------------------------------------------------------------------# # Properties # @@ -314,8 +296,8 @@ def update(self): @property def header(self): self.update() - return self._blocks["header"] + return self._blocks[spec.HEADER] @property def body(self): self.update() - return self._blocks["body"] + return self._blocks[spec.BODY] diff --git a/bof/spec.py b/bof/spec.py index 8d36312..db37cfc 100644 --- a/bof/spec.py +++ b/bof/spec.py @@ -54,6 +54,9 @@ BITSIZES = "bitsizes" # BLOCKS BLOCK = "block" +# GENERIC NAMES +HEADER = "header" +BODY = "body" ############################################################################### # JSON file management functions # diff --git a/examples/cemi_fuzzer.py b/examples/cemi_fuzzer.py index 4e657f3..f44cc8d 100644 --- a/examples/cemi_fuzzer.py +++ b/examples/cemi_fuzzer.py @@ -17,9 +17,8 @@ def connect(ip:str, port:int) -> (knx.KnxNet, int): channel = 0 try: knxnet.connect(ip, port) - connectreq = knx.KnxFrame(type="CONNECT REQUEST") - connectreq.body.connection_request_information.connection_type_code.value = \ - knx.KnxSpec().get_code_id("connection_type_code", "Device Management Connection") + connectreq = knx.KnxFrame(type="CONNECT REQUEST", + connection="DEVICE MANAGEMENT CONNECTION") connectreq.body.control_endpoint.ip_address.value = byte.from_ipv4(knxnet.source[0]) connectreq.body.control_endpoint.port.value = byte.from_int(knxnet.source[1]) connectreq.body.data_endpoint.ip_address.value = byte.from_ipv4(knxnet.source[0]) From 3560eac62707f6a8e1836d931540ca9fb5dba25a Mon Sep 17 00:00:00 2001 From: Lex Date: Thu, 6 Aug 2020 09:54:39 +0200 Subject: [PATCH 18/24] Fixed depends keywork so that it works with all fields --- bof/base.py | 4 ++- bof/frame.py | 74 ++++++++++++++++++++++++------------------- bof/knx/knxframe.py | 4 +-- examples/knx_write.py | 43 +++++++++++++++++++++++++ 4 files changed, 90 insertions(+), 35 deletions(-) create mode 100644 examples/knx_write.py diff --git a/bof/base.py b/bof/base.py index dc6de3e..f76a6fb 100644 --- a/bof/base.py +++ b/bof/base.py @@ -107,4 +107,6 @@ def log(message:str, level:str="INFO") -> bool: def to_property(value:str) -> str: """Replace all non alphanumeric characters in a string with ``_``""" - return sub('[^0-9a-zA-Z]+', '_', value.lower()) + if isinstance(value, str): + return sub('[^0-9a-zA-Z]+', '_', value.lower().strip()) + return value diff --git a/bof/frame.py b/bof/frame.py index 214f544..636410f 100644 --- a/bof/frame.py +++ b/bof/frame.py @@ -137,7 +137,11 @@ class BOFField(object): def __init__(self, **kwargs): self.name = kwargs[spec.NAME] if spec.NAME in kwargs else "" self._value = kwargs[spec.VALUE] if spec.VALUE in kwargs else b'' - self._size = int(kwargs[spec.SIZE]) if spec.SIZE in kwargs else max(1, byte.get_size(self._value)) + if spec.SIZE in kwargs: + self._size = byte.to_int(kwargs[spec.SIZE]) if isinstance(kwargs[spec.SIZE], bytes) \ + else int(kwargs[spec.SIZE]) + else: + self._size = max(1, byte.get_size(self._value)) self._parent = kwargs[PARENT] if PARENT in kwargs else None self._is_length = kwargs[spec.IS_LENGTH] if spec.IS_LENGTH in kwargs else False self._fixed_size = kwargs[spec.F_SIZE] if spec.F_SIZE in kwargs else False @@ -222,7 +226,12 @@ def size(self) -> int: return self._size @size.setter def size(self, size:int): - self._size = size + if isinstance(size, int): + self._size = size + elif isinstance(size, bytes): + self._size = byte.to_int(size) + else: + raise BOFProgrammingError("Size value should be int or bytes.") self._value = byte.resize(self._value, self._size) @property @@ -395,13 +404,9 @@ def build(self, **kwargs): """ if not spec.TYPE in kwargs or kwargs[spec.TYPE] == spec.BLOCK: return + # If values rely on previous content, replace them user_values = kwargs[USER_VALUES] if USER_VALUES in kwargs else {} - value = kwargs[VALUE] if VALUE in kwargs else None - # If values rely on previous content, replace it - for key in kwargs: - if isinstance(kwargs[key], str) and kwargs[key].startswith(spec.DEPENDS): - dependency = kwargs[key].split(spec.DEPENDS)[1] - kwargs[key] = self._get_depends_block(dependency, user_values) + self._get_depends(kwargs, user_values) # Retrieve the template in the JSON file and check it block_template = self._spec.get_block_template(kwargs[spec.TYPE]) if not block_template: @@ -409,8 +414,12 @@ def build(self, **kwargs): if not isinstance(block_template, list): raise BOFProgrammingError("Invalid block format ({0})".format(kwargs[spec.TYPE])) # Create block and fill them (if value) one by one + value = kwargs[VALUE] if VALUE in kwargs else None for item_template in block_template: - item = self.factory(item_template, value=value, user_values=user_values, parent=self) + # First we need to replace the "depends" part (dictionary must be copied) + final_template = self._get_depends(item_template.copy(), user_values) + item = self.factory(final_template, value=value, + user_values=user_values, parent=self) self.append(item) # If value, we extract part of it to fill the item if value: @@ -508,34 +517,35 @@ def _add_property(self, name, pointer:object) -> None: elif len(name) > 0: setattr(self, to_property(name), pointer) - def _get_depends_block(self, field:str, user_values:dict=None): - """If the format of a block depends on the value of a field set - previously, we look for it and choose the appropriate format. - The closest field with such name is used. + def _get_depends(self, template:dict, user_values:dict=None) -> None: + """If a value or the format of a block depends on another field value + for a field set previously, we look for it and choose the appropriate + format. The closest field with such name is used. If the field name + is in the list of existing code, we match the value with a code, else + we return the value directly. - :param name: Name of the field to look for and extract value. + :param template: Dictionary in which to look for values. :raises BOFProgrammingError: If specified field was not found or no association was found. """ - field = to_property(field) - # First look in user-defined parameter values - if user_values: - for key in user_values: - if field == to_property(key): - block = self._spec.get_code_name(key, user_values[key]) - if block: - return block - # Then look in previously-set fields (starting by the closest ones) - if self._parent: - field_list = list(self._parent) + def get_depends_value(value, user_values): + if user_values: + for key in user_values: + if value == to_property(key): + block = self._spec.get_code_name(key, user_values[key]) + return block if block else user_values[key] + field_list = list(self._parent) + list(self) if self._parent else list(self) field_list.reverse() - for frame_field in field_list: - if field == to_property(frame_field.name): - block = self._spec.get_code_name(frame_field.name, frame_field.value) - if block: - return block - raise BOFProgrammingError("Association not found for field {0}".format(field)) - + for field in field_list: + if value == to_property(field.name): + block = self._spec.get_code_name(field.name, field.value) + return block if block else field.value + raise BOFProgrammingError("Association not found for field {0}".format(value)) + for key in template: + if isinstance(template[key], str) and template[key].startswith(spec.DEPENDS): + dependency = to_property(template[key].split(spec.DEPENDS)[1]) + template[key] = get_depends_value(dependency, user_values) + return template #-------------------------------------------------------------------------# # Properties # diff --git a/bof/knx/knxframe.py b/bof/knx/knxframe.py index 2a7abb9..01f6c02 100644 --- a/bof/knx/knxframe.py +++ b/bof/knx/knxframe.py @@ -62,7 +62,7 @@ def __init__(self, filepath:str=None): def get_code_name(self, code:str, identifier) -> str: code = self._get_dict_key(self.codes, code) - if isinstance(identifier, bytes): + if isinstance(identifier, bytes) and code in self.codes: for key in self.codes[code]: if identifier == bytes.fromhex(key): return self.codes[code][key] @@ -274,4 +274,4 @@ def sid(self) -> str: @property def cemi(self) -> str: """Return the type of cemi, if any.""" - self._spec.get_cemi_name(self._blocks[spec.BODY].cemi.message_code) + return self._spec.get_code_name("message code", self._blocks[spec.BODY].cemi.message_code.value) diff --git a/examples/knx_write.py b/examples/knx_write.py new file mode 100644 index 0000000..98e27dc --- /dev/null +++ b/examples/knx_write.py @@ -0,0 +1,43 @@ +from sys import path, argv +path.append('../') + +from bof import knx, BOFNetworkError + +def update_source(knxnet, field): + field.ip_address.value = knxnet.source_address + field.port.value = knxnet.source_port + +def tunnel_connect(knxnet): + tunnel_connect_request = knx.KnxFrame(type="CONNECT_REQUEST", + connection="TUNNELING CONNECTION") + update_source(knxnet, tunnel_connect_request.body.control_endpoint) + update_source(knxnet, tunnel_connect_request.body.data_endpoint) + tunnel_connect_response = knxnet.send_receive(tunnel_connect_request) + return tunnel_connect_response.body.communication_channel_id.value + +def tunnel_disconnect(knxnet, channel): + tunnel_disconnect_request = knx.KnxFrame(type="DISCONNECT_REQUEST") + tunnel_disconnect_request.body.communication_channel_id.value = channel + update_source(knxnet, tunnel_disconnect_request.body.control_endpoint) + tunnel_disconnect_response = knxnet.send_receive(tunnel_disconnect_request) + +def group_write(knxnet, kga, value): + """Write ``value`` to knx group address ``kga`` on ``knxnet``""" + request = knx.KnxFrame(type="CONFIGURATION REQUEST", cemi="L_Data.req") + print(request) + #TODO + +if len(argv) < 2: + print("Usage: {0} IP".format(argv[0])) + exit(-1) + +try: + knxnet = knx.KnxNet() + knxnet.connect(argv[1]) + channel = tunnel_connect(knxnet) + group_write(knxnet, "1/1/1", 1) + tunnel_disconnect(knxnet, channel) +except BOFNetworkError as bne: + print(bne) +finally: + knxnet.disconnect() From ea8bb6bf393ee7a72ae815d12f2b9b068e027c9b Mon Sep 17 00:00:00 2001 From: JulienBedel Date: Thu, 6 Aug 2020 10:39:45 +0200 Subject: [PATCH 19/24] style(bof): Rename get_code_* methods Because codes are in fact dictionaries : - get_code_name -> get_code_value - get_code_id -> get_code_key --- bof/frame.py | 6 +++--- bof/knx/knxframe.py | 8 ++++---- bof/opcua/opcuaframe.py | 7 +++++-- tests/test_knx_frame.py | 12 ++++++------ tests/test_opcua_frame.py | 8 ++++---- 5 files changed, 22 insertions(+), 19 deletions(-) diff --git a/bof/frame.py b/bof/frame.py index 636410f..96844a4 100644 --- a/bof/frame.py +++ b/bof/frame.py @@ -532,13 +532,13 @@ def get_depends_value(value, user_values): if user_values: for key in user_values: if value == to_property(key): - block = self._spec.get_code_name(key, user_values[key]) + block = self._spec.get_code_value(key, user_values[key]) return block if block else user_values[key] field_list = list(self._parent) + list(self) if self._parent else list(self) field_list.reverse() for field in field_list: if value == to_property(field.name): - block = self._spec.get_code_name(field.name, field.value) + block = self._spec.get_code_value(field.name, field.value) return block if block else field.value raise BOFProgrammingError("Association not found for field {0}".format(value)) for key in template: @@ -654,7 +654,7 @@ def __init__(self, block_class:object, **kwargs): user_values = {} for arg, code in self._user_args.items(): if arg in kwargs: - user_values[code] = self._spec.get_code_id(code, kwargs[arg]) + user_values[code] = self._spec.get_code_key(code, kwargs[arg]) value = kwargs[BYTES] if BYTES in kwargs else None # Now build the frame according to what the spec says. for block_template in self._spec.frame: diff --git a/bof/knx/knxframe.py b/bof/knx/knxframe.py index 01f6c02..5f6044c 100644 --- a/bof/knx/knxframe.py +++ b/bof/knx/knxframe.py @@ -60,7 +60,7 @@ def __init__(self, filepath:str=None): filepath = path.join(path.dirname(path.realpath(__file__)), KNXSPECFILE) super().__init__(filepath) - def get_code_name(self, code:str, identifier) -> str: + def get_code_value(self, code:str, identifier) -> str: code = self._get_dict_key(self.codes, code) if isinstance(identifier, bytes) and code in self.codes: for key in self.codes[code]: @@ -73,7 +73,7 @@ def get_code_name(self, code:str, identifier) -> str: return service return None - def get_code_id(self, dict_key:dict, name:str) -> bytes: + def get_code_key(self, dict_key:dict, name:str) -> bytes: name = to_property(name) dict_key = self._get_dict_key(self.codes, dict_key) for key, value in self.codes[dict_key].items(): @@ -267,11 +267,11 @@ def sid(self) -> str: """Return the name associated to the frame's service identifier, or empty string if it is not set. """ - sid = self._spec.get_code_name("service identifier", + sid = self._spec.get_code_value("service identifier", self._blocks[spec.HEADER].service_identifier.value) return sid if sid else str(self._blocks[spec.HEADER].service_identifier.value) @property def cemi(self) -> str: """Return the type of cemi, if any.""" - return self._spec.get_code_name("message code", self._blocks[spec.BODY].cemi.message_code.value) + return self._spec.get_code_value("message code", self._blocks[spec.BODY].cemi.message_code.value) diff --git a/bof/opcua/opcuaframe.py b/bof/opcua/opcuaframe.py index 4a17814..acba1be 100644 --- a/bof/opcua/opcuaframe.py +++ b/bof/opcua/opcuaframe.py @@ -36,7 +36,7 @@ class OpcuaSpec(spec.BOFSpec): spec = opcua.OpcuaSpec() block_template = spec.get_block_template("HEL_BODY") item_template = spec.get_item_template("HEL_BODY", "protocol version") - message_structure = spec.get_code_name("message_type", "HEL") + message_structure = spec.get_code_value("message_type", "HEL") """ def __init__(self): @@ -77,7 +77,7 @@ def get_block_template(self, block_name:str) -> list: """ return self._get_dict_value(self.blocks, block_name) if block_name else None - def get_code_name(self, code:str, identifier) -> str: + def get_code_value(self, code:str, identifier) -> str: """Returns the value associated to an `identifier` inside a `code` association table. See opcua.json + usage example to better understand the association table concept. @@ -98,6 +98,9 @@ def get_code_name(self, code:str, identifier) -> str: return self.codes[code][key] return None + def get_code_key(self, dict_key:dict, name:str) -> bytes: + return name + ############################################################################### # OPC UA FRAME CONTENT # ############################################################################### diff --git a/tests/test_knx_frame.py b/tests/test_knx_frame.py index 341c567..8d0c004 100644 --- a/tests/test_knx_frame.py +++ b/tests/test_knx_frame.py @@ -65,13 +65,13 @@ class Test01KnxSpecTesting(unittest.TestCase): """Test class for KnxSpec public methods.""" def test_01_get_service_id(self): """Test that we can get a service identifier from its name""" - sid = knx.KnxSpec().get_code_id("service identifier", "description request") + sid = knx.KnxSpec().get_code_key("service identifier", "description request") self.assertEqual(sid, b"\x02\x03") def test_02_get_service_name(self): """Test that we can get the name of a service identifier from its id.""" - name = knx.KnxSpec().get_code_name("service identifier", b"\x02\x03") + name = knx.KnxSpec().get_code_value("service identifier", b"\x02\x03") self.assertEqual(name, "DESCRIPTION REQUEST") - name = knx.KnxSpec().get_code_name("service identifier", "DESCRIPTION_REQUEST") + name = knx.KnxSpec().get_code_value("service identifier", "DESCRIPTION_REQUEST") self.assertEqual(name, "DESCRIPTION REQUEST") def test_03_get_template_from_body(self): """Test that we can retrieve the frame template associated to a body name.""" @@ -79,7 +79,7 @@ def test_03_get_template_from_body(self): self.assertEqual(isinstance(template, list), True) def test_04_get_cemi_name(self): """Test that we can retrieve the name of a cEMI from its message code.""" - cemi = knx.KnxSpec().get_code_name("message_code", b"\xfc") + cemi = knx.KnxSpec().get_code_value("message_code", b"\xfc") self.assertEqual(cemi, "PropRead.req") class Test02AdvancedKnxHeaderCrafting(unittest.TestCase): @@ -247,7 +247,7 @@ def test_01_knx_parse_descrresp(self): def test_02_knx_parse_connectresp(self): connectreq = knx.KnxFrame(type="CONNECT_REQUEST") connectreq.body.connection_request_information.cri_connection_type_code.value = \ - knx.KnxSpec().get_code_id("cri connection type code", "Device Management Connection") + knx.KnxSpec().get_code_key("cri connection type code", "Device Management Connection") self.connection.send(connectreq) connectresp = self.connection.receive() channel = connectresp.body.communication_channel_id.value @@ -281,7 +281,7 @@ def test_03_knx_cemi_bitfields_parsing(self): # ConnectReq connectreq = knx.KnxFrame(type="CONNECT REQUEST") connectreq.body.connection_request_information.cri_connection_type_code.value = \ - knx.KnxSpec().get_code_id("cri connection type code", "Device Management Connection") + knx.KnxSpec().get_code_key("cri connection type code", "Device Management Connection") connectreq.body.control_endpoint.ip_address.value = byte.from_ipv4(knxnet.source[0]) connectreq.body.control_endpoint.port.value = byte.from_int(knxnet.source[1]) connectreq.body.data_endpoint.ip_address.value = byte.from_ipv4(knxnet.source[0]) diff --git a/tests/test_opcua_frame.py b/tests/test_opcua_frame.py index 99a013f..283e78c 100644 --- a/tests/test_opcua_frame.py +++ b/tests/test_opcua_frame.py @@ -74,25 +74,25 @@ def test_11_get_association_str_valid(self): """Test that a valid association is returned as expeted""" spec = opcua.OpcuaSpec() spec.load("tests/jsons/valid.json") - message_structure = spec.get_code_name("message_type", "HEL") + message_structure = spec.get_code_value("message_type", "HEL") self.assertEqual(message_structure, "HEL_BODY") def test_12_get_association_str_invalid(self): """Test that an invalid association returns None""" spec = opcua.OpcuaSpec() spec.load("tests/jsons/valid.json") - message_structure = spec.get_code_name("INVALID", "INVALID") + message_structure = spec.get_code_value("INVALID", "INVALID") self.assertEqual(message_structure, None) def test_13_get_association_bytes_valid(self): """Test that a valid association with byte id is returned as expeted""" spec = opcua.OpcuaSpec() spec.load("tests/jsons/valid.json") - message_structure = spec.get_code_name("message_type", b"HEL") + message_structure = spec.get_code_value("message_type", b"HEL") self.assertEqual(message_structure, "HEL_BODY") def test_14_get_association_bytes_invalid(self): """Test that an invalid association with byte id returns None""" spec = opcua.OpcuaSpec() spec.load("tests/jsons/valid.json") - message_structure = spec.get_code_name("INVALID", b"INVALID") + message_structure = spec.get_code_value("INVALID", b"INVALID") self.assertEqual(message_structure, None) class Test02OpcuaField(unittest.TestCase): From cb26bdb707e86e6678c08c057e4abed9314ad245 Mon Sep 17 00:00:00 2001 From: Lex Date: Thu, 6 Aug 2020 14:36:16 +0200 Subject: [PATCH 20/24] Part of L_cEMI implemented --- bof/knx/knxnet.json | 8 +++++--- examples/knx_write.py | 8 +++++++- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/bof/knx/knxnet.json b/bof/knx/knxnet.json index f8637ed..a617510 100644 --- a/bof/knx/knxnet.json +++ b/bof/knx/knxnet.json @@ -128,12 +128,14 @@ "L_cEMI": [ {"name": "additional info length", "type": "field", "size": 1, "default": "00"}, {"name": "additional information", "type": "field", "size": "depends:additional info length"}, - {"name": "frame type, empty, repeat, system broadcast, priority, ack request, confirm", "type": "field", "size": 1, "bitsizes": "1, 1, 1, 1, 2, 1, 1"}, - {"name": "destination address, hop count, extended frame format", "type": "field", "size": 1, "bitsizes": "1, 3, 4"}, + {"name": "frame type, empty, repeat, broadcast type, priority, ack request, confirm", "type": "field", "size": 1, "bitsizes": "1, 1, 1, 1, 2, 1, 1"}, + {"name": "address type, hop count, extended frame format", "type": "field", "size": 1, "bitsizes": "1, 3, 4"}, {"name": "source address", "type": "field", "size": 2}, {"name": "destination address", "type": "field", "size": 2}, {"name": "information length", "type": "field", "size": 1, "default": "01"}, - {"name": "tpci apci", "type": "field", "size": "depends: information length"} + {"name": "tpci", "type": "field", "size": 1}, + {"name": "apci", "type": "field", "size": 2}, + {"name": "data", "type": "field", "size": "depends: information length"} ], "DP_cEMI": [ {"name": "object type", "type": "field", "size": 2}, diff --git a/examples/knx_write.py b/examples/knx_write.py index 98e27dc..198e310 100644 --- a/examples/knx_write.py +++ b/examples/knx_write.py @@ -24,8 +24,14 @@ def tunnel_disconnect(knxnet, channel): def group_write(knxnet, kga, value): """Write ``value`` to knx group address ``kga`` on ``knxnet``""" request = knx.KnxFrame(type="CONFIGURATION REQUEST", cemi="L_Data.req") + request.body.cemi.cemi_data.l_data_req.frame_type.value = 1 + request.body.cemi.cemi_data.l_data_req.repeat.value = 1 + request.body.cemi.cemi_data.l_data_req.broadcast_type.value = 1 + request.body.cemi.cemi_data.l_data_req.address_type.value = 1 + request.body.cemi.cemi_data.l_data_req.hop_count.value = 6 + request.body.cemi.cemi_data.l_data_req.source_address.value = b"\xff\xff" # TODO: 15.15.255 + request.body.cemi.cemi_data.l_data_req.destination_address.value = b"\x09\x01" # TODO: 15.15.255 print(request) - #TODO if len(argv) < 2: print("Usage: {0} IP".format(argv[0])) From 0e9e2a596fc74d56e211a2d2e958ac272584b4f5 Mon Sep 17 00:00:00 2001 From: Lex Date: Fri, 7 Aug 2020 10:52:37 +0200 Subject: [PATCH 21/24] KNX group write finally works --- bof/knx/knxframe.py | 2 + bof/knx/knxnet.json | 22 +++- examples/discover.py | 23 ---- examples/{knx_write.py => knx_group_write.py} | 22 +++- tests/test_knx_frame.py | 114 +++++++++++++----- 5 files changed, 119 insertions(+), 64 deletions(-) delete mode 100644 examples/discover.py rename examples/{knx_write.py => knx_group_write.py} (71%) diff --git a/bof/knx/knxframe.py b/bof/knx/knxframe.py index 5f6044c..a6cae74 100644 --- a/bof/knx/knxframe.py +++ b/bof/knx/knxframe.py @@ -159,6 +159,8 @@ def factory(cls, template, **kwargs) -> object: if USER_VALUES in kwargs and template[spec.NAME] in kwargs[USER_VALUES]: value = kwargs[USER_VALUES][template[spec.NAME]] elif VALUE in kwargs and kwargs[VALUE]: + if isinstance(template[spec.SIZE], bytes): + template[spec.SIZE] = byte.to_int(template[spec.SIZE]) value = kwargs[VALUE][:template[spec.SIZE]] return KnxField(**template, value=value) return cls(**template, **kwargs) diff --git a/bof/knx/knxnet.json b/bof/knx/knxnet.json index a617510..7b7b386 100644 --- a/bof/knx/knxnet.json +++ b/bof/knx/knxnet.json @@ -71,6 +71,19 @@ {"name": "sequence counter", "type": "field", "size": 1}, {"name": "status", "type": "field", "size": 1} ], + "TUNNELING REQUEST": [ + {"name": "structure length", "type": "field", "size": 1, "default": "04"}, + {"name": "communication channel id", "type": "field", "size": 1, "default": "01"}, + {"name": "sequence counter", "type": "field", "size": 1}, + {"name": "reserved", "type": "field", "size": 1}, + {"name": "cEMI", "type": "CEMI"} + ], + "TUNNELING ACK": [ + {"name": "structure length", "type": "field", "size": 1, "is_length": true}, + {"name": "communication channel id", "type": "field", "size": 1, "default": "01"}, + {"name": "sequence counter", "type": "field", "size": 1}, + {"name": "status", "type": "field", "size": 1} + ], "HPAI": [ {"name": "structure length", "type": "field", "size": 1, "is_length": true}, {"name": "host protocol code", "type": "field", "size": 1, "default": "01"}, @@ -133,9 +146,8 @@ {"name": "source address", "type": "field", "size": 2}, {"name": "destination address", "type": "field", "size": 2}, {"name": "information length", "type": "field", "size": 1, "default": "01"}, - {"name": "tpci", "type": "field", "size": 1}, - {"name": "apci", "type": "field", "size": 2}, - {"name": "data", "type": "field", "size": "depends: information length"} + {"name": "packet type, sequence type, empty", "type": "field", "size": 2, "bitsizes": "1, 1, 6"}, + {"name": "service, data", "type": "field", "size": "depends: information length", "bitsizes": "2, 6"} ], "DP_cEMI": [ {"name": "object type", "type": "field", "size": 2}, @@ -177,7 +189,9 @@ "0209": "DISCONNECT REQUEST", "020A": "DISCONNECT RESPONSE", "0310": "CONFIGURATION REQUEST", - "0311": "CONFIGURATION ACK" + "0311": "CONFIGURATION ACK", + "0420": "TUNNELING REQUEST", + "0421": "TUNNELING ACK" }, "message code": { "11": "L_Data.req", diff --git a/examples/discover.py b/examples/discover.py deleted file mode 100644 index 376e01c..0000000 --- a/examples/discover.py +++ /dev/null @@ -1,23 +0,0 @@ -from sys import path, argv -path.append('../') - -from bof import knx, BOFNetworkError - -if len(argv) < 2: - print("Usage: python {0} IP_ADDRESS".format(argv[0])) -else: - knxnet = knx.KnxNet() - ip, port = argv[1], 3671 - try: - knxnet.connect(ip, port) - frame = knx.KnxFrame(type="DESCRIPTION REQUEST") - # print(frame) - knxnet.send(frame) - response = knxnet.receive() - print(response) - device = knx.KnxDevice(response, ip_address=ip, port=port) - print(device) - except BOFNetworkError as bne: - print(str(bne)) - finally: - knxnet.disconnect() diff --git a/examples/knx_write.py b/examples/knx_group_write.py similarity index 71% rename from examples/knx_write.py rename to examples/knx_group_write.py index 198e310..1d05ff6 100644 --- a/examples/knx_write.py +++ b/examples/knx_group_write.py @@ -21,9 +21,9 @@ def tunnel_disconnect(knxnet, channel): update_source(knxnet, tunnel_disconnect_request.body.control_endpoint) tunnel_disconnect_response = knxnet.send_receive(tunnel_disconnect_request) -def group_write(knxnet, kga, value): +def group_write(knxnet, channel, kga, value): """Write ``value`` to knx group address ``kga`` on ``knxnet``""" - request = knx.KnxFrame(type="CONFIGURATION REQUEST", cemi="L_Data.req") + request = knx.KnxFrame(type="TUNNELING REQUEST", cemi="L_Data.req") request.body.cemi.cemi_data.l_data_req.frame_type.value = 1 request.body.cemi.cemi_data.l_data_req.repeat.value = 1 request.body.cemi.cemi_data.l_data_req.broadcast_type.value = 1 @@ -31,17 +31,27 @@ def group_write(knxnet, kga, value): request.body.cemi.cemi_data.l_data_req.hop_count.value = 6 request.body.cemi.cemi_data.l_data_req.source_address.value = b"\xff\xff" # TODO: 15.15.255 request.body.cemi.cemi_data.l_data_req.destination_address.value = b"\x09\x01" # TODO: 15.15.255 - print(request) + request.body.cemi.cemi_data.l_data_req.service.value = 2 + request.body.cemi.cemi_data.l_data_req.data.value = value + received_ack = knxnet.send_receive(request) + print(received_ack) + response = knxnet.receive() + print(response.cemi) + if response.sid == "TUNNELING REQUEST": + ack_to_send = knx.KnxFrame(type="TUNNELING ACK") + ack_to_send.body.communication_channel_id.value = channel + knxnet.send(ack_to_send) + print(response) -if len(argv) < 2: - print("Usage: {0} IP".format(argv[0])) +if len(argv) < 4: + print("Usage: {0} IP_ADDRESS KNX_GROUPADDRESS VALUE".format(argv[0])) exit(-1) try: knxnet = knx.KnxNet() knxnet.connect(argv[1]) channel = tunnel_connect(knxnet) - group_write(knxnet, "1/1/1", 1) + group_write(knxnet, channel, argv[2], int(argv[3])) tunnel_disconnect(knxnet, channel) except BOFNetworkError as bne: print(bne) diff --git a/tests/test_knx_frame.py b/tests/test_knx_frame.py index 8d0c004..8347b9c 100644 --- a/tests/test_knx_frame.py +++ b/tests/test_knx_frame.py @@ -274,48 +274,100 @@ def test_02_knx_cemi_bitfields(self): self.assertEqual(frame.body.cemi.cemi_data.propread_req.number_of_elements.value, [1,1,1,1]) self.assertEqual(frame.body.cemi.cemi_data.propread_req.start_index.value, [0,0,0,0,0,0,0,0,0,0,0,1]) self.assertEqual(frame.body.cemi.cemi_data.propread_req.number_of_elements_start_index.value, b'\xF0\x01') - def test_03_knx_cemi_bitfields_parsing(self): - """Test that a received cEMI frame with bit fields is parsed.""" - knxnet = knx.KnxNet() - knxnet.connect(BOIBOITE, 3671) + +class Test06cEMIConfigFrame(unittest.TestCase): + def setUp(self): + def update_source(knxnet, field): + field.ip_address.value = knxnet.source_address + field.port.value = knxnet.source_port + self.connection = knx.KnxNet() + self.connection.connect(BOIBOITE, 3671) # ConnectReq - connectreq = knx.KnxFrame(type="CONNECT REQUEST") - connectreq.body.connection_request_information.cri_connection_type_code.value = \ - knx.KnxSpec().get_code_key("cri connection type code", "Device Management Connection") - connectreq.body.control_endpoint.ip_address.value = byte.from_ipv4(knxnet.source[0]) - connectreq.body.control_endpoint.port.value = byte.from_int(knxnet.source[1]) - connectreq.body.data_endpoint.ip_address.value = byte.from_ipv4(knxnet.source[0]) - connectreq.body.data_endpoint.port.value = byte.from_int(knxnet.source[1]) + connectreq = knx.KnxFrame(type="CONNECT REQUEST", + connection="Device Management Connection") + update_source(self.connection, connectreq.body.control_endpoint) + update_source(self.connection, connectreq.body.data_endpoint) #ConnectResp - connectresp = knxnet.send_receive(connectreq) - channel = connectresp.body.communication_channel_id.value + connectresp = self.connection.send_receive(connectreq) + self.channel = connectresp.body.communication_channel_id.value + def tearDown(self): + def update_source(knxnet, field): + field.ip_address.value = knxnet.source_address + field.port.value = knxnet.source_port + discoreq = knx.KnxFrame(type="DISCONNECT_REQUEST") + discoreq.body.communication_channel_id.value = self.channel + update_source(self.connection, discoreq.body.control_endpoint) + discoresp = self.connection.send_receive(discoreq) + self.connection.disconnect() + def test_01_knx_cemi_bitfields_parsing(self): + """Test that a received cEMI frame with bit fields is parsed.""" #ConfigReq request = knx.KnxFrame(type="CONFIGURATION REQUEST", cemi="PropRead.req") - request.body.communication_channel_id.value = channel + request.body.communication_channel_id.value = self.channel request.body.cemi.cemi_data.propread_req.number_of_elements.value = 1 request.body.cemi.cemi_data.propread_req.object_type.value = 11 request.body.cemi.cemi_data.propread_req.property_id.value = 53 # Ack + ConfigReq response - response = knxnet.send_receive(request) # ACK + response = self.connection.send_receive(request) # ACK while (1): - response = knxnet.receive() # PropRead.con + response = self.connection.receive() # PropRead.con if response.sid == "CONFIGURATION REQUEST": # TEST SUBFIELDS - self.assertEqual(byte.bit_list_to_int(response.body.cemi.cemi_data.propread_con.number_of_elements.value), 0) - self.assertEqual(byte.bit_list_to_int(response.body.cemi.cemi_data.propread_con.start_index.value), 0) - response.body.cemi.cemi_data.propread_con.number_of_elements_start_index.value = b'\x10\x01' - self.assertEqual(byte.bit_list_to_int(response.body.cemi.cemi_data.propread_con.number_of_elements.value), 1) - self.assertEqual(byte.bit_list_to_int(response.body.cemi.cemi_data.propread_con.start_index.value), 1) + propread_con = response.body.cemi.cemi_data.propread_con + self.assertEqual(byte.bit_list_to_int(propread_con.number_of_elements.value), 0) + self.assertEqual(byte.bit_list_to_int(propread_con.start_index.value), 0) + propread_con.number_of_elements_start_index.value = b'\x10\x01' + self.assertEqual(byte.bit_list_to_int(propread_con.number_of_elements.value), 1) + self.assertEqual(byte.bit_list_to_int(propread_con.start_index.value), 1) # We tell the boiboite we received it ack = knx.KnxFrame(type="CONFIGURATION ACK") - ack.body.communication_channel_id.value = channel - knxnet.send(ack) + ack.body.communication_channel_id.value = self.channel + self.connection.send(ack) break - # DisconnectReq - discoreq = knx.KnxFrame(type="DISCONNECT REQUEST") - discoreq.body.communication_channel_id.value = channel - discoreq.body.control_endpoint.ip_address.value = byte.from_ipv4(knxnet.source[0]) - discoreq.body.control_endpoint.port.value = byte.from_int(knxnet.source[1]) - # DisconnectResp - knxnet.send(discoreq) - knxnet.disconnect() + +class Test07cEMITunnelFrame(unittest.TestCase): + def setUp(self): + def update_source(knxnet, field): + field.ip_address.value = knxnet.source_address + field.port.value = knxnet.source_port + self.connection = knx.KnxNet() + self.connection.connect(BOIBOITE, 3671) + # ConnectReq + connectreq = knx.KnxFrame(type="CONNECT REQUEST", + connection="Tunneling Connection") + update_source(self.connection, connectreq.body.control_endpoint) + update_source(self.connection, connectreq.body.data_endpoint) + #ConnectResp + connectresp = self.connection.send_receive(connectreq) + self.channel = connectresp.body.communication_channel_id.value + def tearDown(self): + def update_source(knxnet, field): + field.ip_address.value = knxnet.source_address + field.port.value = knxnet.source_port + discoreq = knx.KnxFrame(type="DISCONNECT_REQUEST") + discoreq.body.communication_channel_id.value = self.channel + update_source(self.connection, discoreq.body.control_endpoint) + discoresp = self.connection.send_receive(discoreq) + self.connection.disconnect() + def test_01_knx_cemi_datareq_working(self): + """Test that a received cEMI frame with bit fields is parsed.""" + request = knx.KnxFrame(type="TUNNELING REQUEST", cemi="L_Data.req") + request.body.cemi.cemi_data.l_data_req.frame_type.value = 1 + request.body.cemi.cemi_data.l_data_req.repeat.value = 1 + request.body.cemi.cemi_data.l_data_req.broadcast_type.value = 1 + request.body.cemi.cemi_data.l_data_req.address_type.value = 1 + request.body.cemi.cemi_data.l_data_req.hop_count.value = 6 + request.body.cemi.cemi_data.l_data_req.source_address.value = b"\xff\xff" # TODO: 15.15.255 + request.body.cemi.cemi_data.l_data_req.destination_address.value = b"\x09\x01" # TODO: 15.15.255 + request.body.cemi.cemi_data.l_data_req.service.value = 2 + request.body.cemi.cemi_data.l_data_req.data.value = 1 + received_ack = self.connection.send_receive(request) + self.assertEqual(received_ack.body.status.value, b'\x00') + response = self.connection.receive() + if response.sid == "TUNNELING REQUEST": + ack_to_send = knx.KnxFrame(type="TUNNELING ACK") + ack_to_send.body.communication_channel_id.value = self.channel + self.connection.send(ack_to_send) + l_data_con = response.body.cemi.cemi_data.l_data_con + self.assertEqual(l_data_con.destination_address.value, b'\x09\x01') + self.assertEqual(l_data_con.service_data.value, b'\x81') From 8d3e7f8e851e8d5f9ac5655a2e817c13c5088f76 Mon Sep 17 00:00:00 2001 From: Lex Date: Fri, 7 Aug 2020 12:15:04 +0200 Subject: [PATCH 22/24] KNX address conversion and unittest for new KNX stuff --- bof/byte.py | 48 +++++++++++++++++++++++++++++++++++++ bof/knx/knxdevice.py | 8 +------ bof/knx/knxframe.py | 4 +++- examples/knx_group_write.py | 8 +++---- tests/test_byte.py | 17 +++++++++++++ tests/test_knx_frame.py | 2 +- 6 files changed, 73 insertions(+), 14 deletions(-) diff --git a/bof/byte.py b/bof/byte.py index 42a58db..c3dd595 100644 --- a/bof/byte.py +++ b/bof/byte.py @@ -9,6 +9,8 @@ as first parameter. """ +from re import match + from .base import BOFProgrammingError from ipaddress import IPv4Address @@ -178,6 +180,52 @@ def from_mac(mac:str): array = bytes.fromhex(mac.replace(":", "")) return array +def to_knx(value:bytes, group=False) -> str: + """Converts a 2-bytes array to a KNX individual (X.Y.Z) or group + address (X/Y/Z). + + :Individual address ``X.Y.Z``: X and Y are 4 bits (1st byte) and + Z is 8 bits (2nd byte). + :Group address ``X/Y/Z``: X is 6 bits, Y is 3 bits (1st byte) and + Z is 8 bits (2nd byte). + + :param value: Byte array (2 bytes) to convert + :param group: Boolean stating if the KNX address is a group address + (default is False: the final string will have a the + format of an individual KNX address. + """ + if not len(value): + return None + first_chunk_size = 5 if group else 4 + string_format = "{0}/{1}/{2}" if group else "{0}.{1}.{2}" + bitlist = to_bit_list(value[:1]) + x = int("".join([str(x) for x in bitlist[:first_chunk_size]]), 2) + y = int("".join([str(x) for x in bitlist[first_chunk_size:]]), 2) + z = to_int(value[1:]) + return string_format.format(x, y, z) + +def from_knx(address:str) -> bytes: + """Converts a KNX individual address (X.Y.Z) or group address + (X/Y/Z) to a byte array (2 bytes, X Y being one byte, and Z the + other). + + :param address: KNX address as a string with format X.Y.Z or + X/Y/Z. + """ + addr = match("(\d{1,2})\.(\d{1,2})\.(\d{1,3})", address) + first_chunk_size = 4 # individual address + if not addr: + addr = match("(\d{1,2})/(\d{1,2})/(\d{1,3})", address) + first_chunk_size = 5 # group address + if not addr: + return None + x, y, z = (int(addr.group(a+1)) for a in range(3)) + x = int_to_bit_list(x)[8-first_chunk_size:] + y = int_to_bit_list(y)[first_chunk_size:] + b1 = from_int(bit_list_to_int(x + y)) + b2 = from_int(z) + return b''.join([b1, b2]) + def int_to_bit_list(n:int, size:int=8, byteorder:str=None) -> list: """Representation of n as a list of bits (0 or 1). diff --git a/bof/knx/knxdevice.py b/bof/knx/knxdevice.py index effb01e..5f43b97 100644 --- a/bof/knx/knxdevice.py +++ b/bof/knx/knxdevice.py @@ -210,13 +210,7 @@ def knx_address(self) -> str: return self.__knx_address @knx_address.setter def knx_address(self, value:str): - if len(value): - bitlist = byte.to_bit_list(value[:1]) - x = int("".join([str(x) for x in bitlist[:4]]), 2) - y = int("".join([str(x) for x in bitlist[4:]]), 2) - z = byte.to_int(value[1:]) - value = "{0}.{1}.{2}".format(x, y, z) - self.__knx_address = value + self.__knx_address = byte.to_knx(value) @property def port(self) -> str: return self.__port diff --git a/bof/knx/knxframe.py b/bof/knx/knxframe.py index a6cae74..dd03c33 100644 --- a/bof/knx/knxframe.py +++ b/bof/knx/knxframe.py @@ -115,7 +115,9 @@ def value(self, content) -> None: ip_address(content) content = byte.from_ipv4(content) except ValueError: - pass + # Check if content is a KNX address (X.Y.Z or X/Y/Z) + knx_addr = byte.from_knx(content) + content = knx_addr if knx_addr else content super(KnxField, self.__class__).value.fset(self, content) #-----------------------------------------------------------------------------# diff --git a/examples/knx_group_write.py b/examples/knx_group_write.py index 1d05ff6..310d46e 100644 --- a/examples/knx_group_write.py +++ b/examples/knx_group_write.py @@ -29,14 +29,12 @@ def group_write(knxnet, channel, kga, value): request.body.cemi.cemi_data.l_data_req.broadcast_type.value = 1 request.body.cemi.cemi_data.l_data_req.address_type.value = 1 request.body.cemi.cemi_data.l_data_req.hop_count.value = 6 - request.body.cemi.cemi_data.l_data_req.source_address.value = b"\xff\xff" # TODO: 15.15.255 - request.body.cemi.cemi_data.l_data_req.destination_address.value = b"\x09\x01" # TODO: 15.15.255 + request.body.cemi.cemi_data.l_data_req.source_address.value = "15.15.255" + request.body.cemi.cemi_data.l_data_req.destination_address.value = kga request.body.cemi.cemi_data.l_data_req.service.value = 2 request.body.cemi.cemi_data.l_data_req.data.value = value received_ack = knxnet.send_receive(request) - print(received_ack) response = knxnet.receive() - print(response.cemi) if response.sid == "TUNNELING REQUEST": ack_to_send = knx.KnxFrame(type="TUNNELING ACK") ack_to_send.body.communication_channel_id.value = channel @@ -44,7 +42,7 @@ def group_write(knxnet, channel, kga, value): print(response) if len(argv) < 4: - print("Usage: {0} IP_ADDRESS KNX_GROUPADDRESS VALUE".format(argv[0])) + print("Usage: {0} IP_ADDRESS KNX_ADDRESS VALUE".format(argv[0])) exit(-1) try: diff --git a/tests/test_byte.py b/tests/test_byte.py index 802b130..5596157 100644 --- a/tests/test_byte.py +++ b/tests/test_byte.py @@ -190,6 +190,23 @@ def test_05_bytes_to_mac_less_6_bytes(self): result = bof.to_mac(b'\xAB\x5C') self.assertEqual("AB:5C".upper(),result.upper()) +class Test07BytesMacConversion(unittest.TestCase): + """Test class for bytes conversion to mac address.""" + @classmethod + def setUpClass(self): + bof.set_byteorder('big') + def test_01_knx_individual_to_bytes(self): + """Test that we can convert a KNX address with format X.Y.Z""" + self.assertEqual(bof.byte.from_knx("15.15.255"), b"\xff\xff") + def test_02_bytes_to_knx_individual(self): + """Test that we can convert bytes to X.Y.Z""" + self.assertEqual(bof.byte.to_knx(b"\x11\x02"), "1.1.2") + def test_03_knx_group_to_bytes(self): + """Test that we can convert KNX address with format X/Y/Z""" + self.assertEqual(bof.byte.from_knx("1/1/1"), b"\x09\x01") + def test_02_bytes_to_knx_group(self): + """Test that we can convert bytes to X/Y/Z""" + self.assertEqual(bof.byte.to_knx(b"\x09\xff", group=True), "1/1/255") if __name__ == '__main__': unittest.main() diff --git a/tests/test_knx_frame.py b/tests/test_knx_frame.py index 8347b9c..be1a20b 100644 --- a/tests/test_knx_frame.py +++ b/tests/test_knx_frame.py @@ -350,7 +350,7 @@ def update_source(knxnet, field): discoresp = self.connection.send_receive(discoreq) self.connection.disconnect() def test_01_knx_cemi_datareq_working(self): - """Test that a received cEMI frame with bit fields is parsed.""" + """Test that a received cEMI frame after a group write is correct.""" request = knx.KnxFrame(type="TUNNELING REQUEST", cemi="L_Data.req") request.body.cemi.cemi_data.l_data_req.frame_type.value = 1 request.body.cemi.cemi_data.l_data_req.repeat.value = 1 From bf793dbdaf5b917973afe4931d93d2ec54dacefb Mon Sep 17 00:00:00 2001 From: Lex Date: Fri, 7 Aug 2020 17:03:16 +0200 Subject: [PATCH 23/24] Preparing for release 0.3.0 before branching --- CHANGELOG.md | 13 +++++++++++++ docs/man/dev.rst | 46 ++++++++++++++++++++++++++++++++++++---------- 2 files changed, 49 insertions(+), 10 deletions(-) create mode 100644 CHANGELOG.md diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..80a41fe --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,13 @@ +Changelog +========= + +Release 0.3.0 +------------- + +* [BOF] Generic block and frame code moved from KNX and OPCUA + implementations to core +* [BOF] New "depends:" keyword in JSON specification file, to make the + value of a field depend on the value of another one +* [BOF] First version of developer documentation +* [KNX] Tunneling connection management with ``L_Data`` messages +* [KNX] KNX individual and group address parsing and conversion diff --git a/docs/man/dev.rst b/docs/man/dev.rst index ad931ff..2e990e1 100644 --- a/docs/man/dev.rst +++ b/docs/man/dev.rst @@ -337,16 +337,42 @@ bytes arrays, which contain blocks, which contain fields. If they don't, you can skip this part. Otherwise, your protocol implementation should include three classes, inheriting from ``BOFFrame``, ``BOFBlock`` and ``BOFField``. -Formats and behavior that do not match with what is decribed above (mostly, JSON -specification file organization) have to be written to your protocol -implementation's subclasses. +Most of the creation and parsing operation is handled directly by these BOF +classes, relying on what you wrote in a JSON file. Therefore: + +- Block and frame require that you instantiate a specification object inheriting + from ``BOFSpec`` in your subclass' constructor (``__init__``) prior to calling + their init. For frames, you also need to specify the type of block to + instantiate (BOF cannot guess :(). For instance:: + + self._spec = KnxSpec() + super().__init__(KnxBlock, **kwargs) + self.update() + +- All protocol-specific content must be added to your subclass, most probably by + overloading ``BOFFrame``, ``BOFBlock`` and ``BOFField`` methods and + properties. As an example, the setter for the attribute ``value`` in + ``KnxField`` (inheriting from ``BOFField``) has been modified to handle and + convert to bytes IPv4 addresses and KNX individual and group addresses:: + + @value.setter + def value(self, content) -> None: + if isinstance(content, str): + # Check if content is an IPv4 address (A.B.C.D): + try: + ip_address(content) + content = byte.from_ipv4(content) + except ValueError: + # Check if content is a KNX address (X.Y.Z or X/Y/Z) + knx_addr = byte.from_knx(content) + content = knx_addr if knx_addr else content + super(KnxField, self.__class__).value.fset(self, content) .. note:: - So far (BOF v0.2.X), part of the code that we expect to be generic and used - by most of the implementation is not written to BOF core, but to the KNX - implementation. We are carefully moving them as we notice that they can be - reused, but this is a long process and we don't want to miss steps. So far, - please refer to KNX's frame, block and field implementations in - ``bof/knx/knxframe.py`` to write your own implementation and feel free to try - and move part of it to the core (``bof/frame.py``). + The generic part of BOF's frames implementation has been written + according to two protocol implementation (KNX and OPCUA). There may be + some improvement to make (adding parts that are currently written + directly to implementation in the core or removing parts that are not + generic enough) and we count on you to let us know (or make the change + yourself)! From c94e6e07bc67e481b4a2694afdcd5dc0ba961480 Mon Sep 17 00:00:00 2001 From: Lex Date: Fri, 7 Aug 2020 17:06:40 +0200 Subject: [PATCH 24/24] Release 0.3.0 ready --- bof/opcua/__init__.py | 21 --- bof/opcua/opcua.json | 30 ---- bof/opcua/opcuaframe.py | 306 -------------------------------------- docs/conf.py | 2 +- setup.py | 2 +- tests/jsons/invalid.json | 1 - tests/jsons/valid.json | 30 ---- tests/test_opcua_frame.py | 185 ----------------------- 8 files changed, 2 insertions(+), 575 deletions(-) delete mode 100644 bof/opcua/__init__.py delete mode 100644 bof/opcua/opcua.json delete mode 100644 bof/opcua/opcuaframe.py delete mode 100644 tests/jsons/invalid.json delete mode 100644 tests/jsons/valid.json delete mode 100644 tests/test_opcua_frame.py diff --git a/bof/opcua/__init__.py b/bof/opcua/__init__.py deleted file mode 100644 index 394c8aa..0000000 --- a/bof/opcua/__init__.py +++ /dev/null @@ -1,21 +0,0 @@ -""" -OPC UA protocol implementation for industrial automation. - -The implementation schemes and frame descriptions are based on **IEC 62541** -standard, **v1.04**. The specification is described in :file:`opcua.json`. - -A user should be able to create or alter any frame to both valid and invalid -format. Therefore, a user can copy, modify or replace the specification file to -include or change any content they want. - -The ``opcua`` submodule has the following content: - -:opcuaframe: - Object representations of OPC UA frames and frame content (blocks and - fields) and specification details, with methods to build, alter or read - a frame or part of it. Available from direct import of the ``opcua`` - submodule (``from bof import opcua``). -""" - -from .opcuaframe import * - diff --git a/bof/opcua/opcua.json b/bof/opcua/opcua.json deleted file mode 100644 index db82326..0000000 --- a/bof/opcua/opcua.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "frame": [ - {"name": "header", "type": "HEADER"}, - {"name": "body", "type": "depends:message_type"} - ], - "blocks": { - "HEADER": [ - {"name": "message_type", "type": "field", "size": 3}, - {"name": "is_final", "type": "field", "size": 1}, - {"name": "message_size", "type": "field", "size": 4} - ], - "HEL_BODY": [ - {"name": "protocol_version", "type": "field", "size": 4}, - {"name": "receive_buffer_size", "type": "field", "size": 4}, - {"name": "send_buffer_size", "type": "field", "size": 4}, - {"name": "max_message_size", "type": "field", "size": 4}, - {"name": "max_chunk_count", "type": "field", "size": 4}, - {"name": "endpoint_url", "type": "STRING"} - ], - "STRING": [ - {"name": "string_length", "type": "field", "size": 4}, - {"name": "string_value", "type": "field", "size": 14} - ] - }, - "codes" : { - "message_type": { - "HEL": "HEL_BODY" - } - } -} \ No newline at end of file diff --git a/bof/opcua/opcuaframe.py b/bof/opcua/opcuaframe.py deleted file mode 100644 index acba1be..0000000 --- a/bof/opcua/opcuaframe.py +++ /dev/null @@ -1,306 +0,0 @@ -""" -OPC UA frame handling ------------------- - -OPC UA frames handling implementation, implementing ``bof.frame``'s -``BOFSpec``, ``BOFFrame``, ``BOFBlock`` and ``BOFField`` classes. -See `bof/frame.py`. - -BOF should not contain code that is bound to a specific version of a protocol's -specifications. Therefore OPC UA frame structure and its content is described in -:file:`opcua.json`. -""" - -from os import path - -from ..base import BOFProgrammingError, to_property, log -from ..frame import BOFFrame, BOFBlock, BOFField, USER_VALUES, VALUE -from .. import byte, spec - -############################################################################### -# OPCUA SPECIFICATION CONTENT # -############################################################################### - -OPCUASPECFILE = "opcua.json" - -class OpcuaSpec(spec.BOFSpec): - """Singleton class for OPC UA specification content usage. - Inherits ``BOFSpec``, see `bof/frame.py`. - - The default specification is ``opcua.json`` however the end user is free - to modify this file (add categories, contents and attributes) or create a - new file following this format. - - Usage example:: - - spec = opcua.OpcuaSpec() - block_template = spec.get_block_template("HEL_BODY") - item_template = spec.get_item_template("HEL_BODY", "protocol version") - message_structure = spec.get_code_value("message_type", "HEL") - """ - - def __init__(self): - """Initialize the specification object with a JSON file. - If `filepath` is not specified, we use a default one specified in - `OPCUASPECFILE`. - """ - #TODO: add support for custom file path (depends on BOFSpec) - filepath = path.join(path.dirname(path.realpath(__file__)), OPCUASPECFILE) - super().__init__(filepath) - - #-------------------------------------------------------------------------# - # Public # - #-------------------------------------------------------------------------# - - def get_item_template(self, block_name:str, item_name:str) -> dict: - """Returns an item template (dict of values) from a `block_name` and - a `field_name`. - - Note that an item template can represent either a field or a block, - depending on the "type" key of the item. - - :param block_name: Name of the block containing the item. - :param item_name: Name of the item to look for in the block. - :returns: Item template associated to block_name and item_name - """ - if block_name in self.blocks: - for item in self.blocks[block_name]: - if item['name'] == item_name: - return item - return None - - def get_block_template(self, block_name:str) -> list: - """Returns a block template (list of item templates) from a block name. - - :param block_name: Name of the block we want the template from. - :returns: Block template associated with the specifified block_name. - """ - return self._get_dict_value(self.blocks, block_name) if block_name else None - - def get_code_value(self, code:str, identifier) -> str: - """Returns the value associated to an `identifier` inside a - `code` association table. See opcua.json + usage - example to better understand the association table concept. - - :param identifier: Key we want the value from, as str or byte. - :param code: Association table name we want to look into - for identifier match. - :returns: value associated to an identifier inside a code. - """ - code = self._get_dict_key(self.codes, code) - if isinstance(identifier, bytes) and code in self.codes: - for key in self.codes[code]: - if identifier == str.encode(key): - return self.codes[code][key] - elif isinstance(identifier, str) and code in self.codes: - for key in self.codes[code]: - if identifier == key: - return self.codes[code][key] - return None - - def get_code_key(self, dict_key:dict, name:str) -> bytes: - return name - -############################################################################### -# OPC UA FRAME CONTENT # -############################################################################### - -#-----------------------------------------------------------------------------# -# OPC UA fields (byte or byte array) representation # -#-----------------------------------------------------------------------------# - -class OpcuaField(BOFField): - """An ``OpcuaField`` is a set of raw bytes with a name, a size and a - content (``value``). Inherits ``BOFField``, see `bof/frame.py`. - - Usage example:: - - # creating a field from parameters - field = opcua.OpcuaField(name="protocol version", value=b"1", size=4) - - # creating a field from a template - item_template_field = spec.get_item_template("HEL_BODY", "protocol version") - field = opcua.OpcuaField(**item_template_field) - - # editing field value - field.value = b'\x00\x00\x00\x02' - """ - - # For now there is no need to redefine getter and setter property from - # BOFField (base class attributes are compatible with OPC UA field spec) - -#-----------------------------------------------------------------------------# -# OPC UA blocks (set of fields) representation # -#-----------------------------------------------------------------------------# - -class OpcuaBlock(BOFBlock): - """Object representation of an OPC UA block. Inherits ``BOFBlock``. - See `bof/frame.py`. - - An OpcuaBlock (as well as a BOFBlock) contains a set of items. - Those items can be fields as well as blocks, therefore creating - so-called "nested blocks" (or "sub-block"). - - A block is usually built from a template which gives its structure. - Bytes can also be specified to "fill" this block stucture. If the bytes - values are coherent, the structure can also be determined directly from - them. - If no structure is specified the block remains empty. - - Some block field value (typically a sub-block type) may depend on the - value of another field. In that case the keyword "depends:" is used to - associate the variable to its value, based on given parameters to another - field. The process can be looked in details in the `__init__` method, and - understood from examples. - - Usage example:: - - # empty block creation - block = opcua.OpcuaBlock(name="empty_block") - - # block creation with direct parameters - block = opcua.OpcuaBlock(name="header", type="HEADER") - - # block creation from an item template (as found in json spec file) - item_template_block = {"name": "header", "type": "HEADER"} - block = opcua.OpcuaBlock(**item_template_block) - - # fills block with byte value at creation - (note that here a type is still mandatory) - block = opcua.OpcuaBlock(type="STRING", value=14*b"\x01") - - # a block with dependency can be created from raw bytes (no default parameter) - data1 = b'HEL\x00...' - data2 = b'\x00...' - block = opcua.OpcuaBlock() - block.append(opcua.OpcuaBlock(value=data1, parent=block, - **{"name": "header", "type": "HEADER"})) - block.append(opcua.OpcuaBlock(alue=data2, parent=block, - **{"name": "body", "type": "depends:message_type"})) - # this in an example of block looking in its sibling for dependency value - - # we can access a list of available block `fields` using : - block.attributes - - # and access one of those `fields` with : - block.field_name - """ - - @classmethod - def factory(cls, item_template:dict, **kwargs) -> object: - """Returns either an `OpcuaBlock` or an `OpcuaField` depending on the - template specified item type. That's why it's a factory as a class method. - - :param item_template: item template representing sub-block or field. - :returns: A new instance of an OpcuaBlock or an OpcuaField. - - Keyword arguments: - - :param user_values: Default values to assign a field as dictionnary. - :param value: Bytes value to fill the item (block or field) with. - """ - # case where item template represents a field (non-recursive) - if spec.TYPE in item_template and item_template[spec.TYPE] == spec.FIELD: - value = b'' - if USER_VALUES in kwargs and item_template[spec.NAME] in kwargs[USER_VALUES]: - value = kwargs[USER_VALUES][item_template[spec.NAME]] - elif VALUE in kwargs and kwargs[VALUE]: - value = kwargs[VALUE][:item_template[spec.SIZE]] - return OpcuaField(**item_template, value=value) - # case where item template represents a sub-block (nested/recursive block) - else: - return cls(**item_template, **kwargs) - - def __init__(self, **kwargs): - """Initialize the ``OpcuaBlock``. - - Keyword arguments: - - :param type: a string specifying block type (as found in json - specifications) to construct the block on. - :param user_values: default values to assign a field as dictionnary - (can therefore be used to construct blocks with - dependencies if not found in raw bytes, see example - above) - :param value: bytes value to fill the block with - (can create dependencies on its own, see example - above). If user_values parameter is found it overcomes - the value passed as bytes. - - See example in class docstring to understand dependency creation - either with user_values or value parameter. - - """ - self._spec = OpcuaSpec() - super().__init__(**kwargs) - -#-----------------------------------------------------------------------------# -# OPC UA frames representation # -#-----------------------------------------------------------------------------# - -class OpcuaFrame(BOFFrame): - """Object representation of an OPC UA frame, created from the template - in `opcua.json`. - - Uses various initialization methods to create a frame : - - :Byte array: Build the object from a raw byte array, typically used - when receiving incoming connection. In this case block - dependencies are identified automatically. - :Keyword arguments: Uses keyword described in __user_values to fill frame - fields. - - Usage example:: - - # creation from raw bytes (format is automatically identified) - data = b'HEL\x00..' - frame = opcua.OpcuaFrame(bytes=data) - - # creation from known type (who is actually a needed dependence) - # in order to create the frame (see frame structure in opcua.json) - frame = opcua.OpcuaFrame(type="HEL") - """ - - _user_args = { - # {Argument name: field name} - "type": "message_type", - } - - def __init__(self, **kwargs): - """Initialize an OpcuaFrame from various origins using values from - keyword arguments : - - :param byte: raw byte array used to build a frame. - :param user_values: every element of __default specifies arguments - that can be passed in order to set fields values - at frame creation. - """ - self._spec = OpcuaSpec() - super().__init__(OpcuaBlock, **kwargs) - self.update() - - #-------------------------------------------------------------------------# - # Public # - #-------------------------------------------------------------------------# - - def update(self): - """Update ``message_size`` field in header according to total - frame length. - """ - #super().update() - if "message_size" in self._blocks[spec.HEADER].attributes: - total = sum([len(block) for block in self._blocks.values()]) - self._blocks[spec.HEADER].message_size._update_value(byte.from_int(total)) - - #-------------------------------------------------------------------------# - # Properties # - #-------------------------------------------------------------------------# - - @property - def header(self): - self.update() - return self._blocks[spec.HEADER] - @property - def body(self): - self.update() - return self._blocks[spec.BODY] diff --git a/docs/conf.py b/docs/conf.py index b6886da..9dd95a7 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -22,7 +22,7 @@ author = 'Lex' # The full version, including alpha/beta/rc tags -release = '0.2.0' +release = '0.3.0' # -- General configuration --------------------------------------------------- diff --git a/setup.py b/setup.py index c677519..4864d8c 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setuptools.setup( name="boiboite-opener-framework", - version="0.2.0", + version="0.3.0", author="Claire Vacherot", author_email="claire.vacherot@orange.com", description="Industrial network protocols communication and packet crafting framework", diff --git a/tests/jsons/invalid.json b/tests/jsons/invalid.json deleted file mode 100644 index b9669c0..0000000 --- a/tests/jsons/invalid.json +++ /dev/null @@ -1 +0,0 @@ -{{} \ No newline at end of file diff --git a/tests/jsons/valid.json b/tests/jsons/valid.json deleted file mode 100644 index db82326..0000000 --- a/tests/jsons/valid.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "frame": [ - {"name": "header", "type": "HEADER"}, - {"name": "body", "type": "depends:message_type"} - ], - "blocks": { - "HEADER": [ - {"name": "message_type", "type": "field", "size": 3}, - {"name": "is_final", "type": "field", "size": 1}, - {"name": "message_size", "type": "field", "size": 4} - ], - "HEL_BODY": [ - {"name": "protocol_version", "type": "field", "size": 4}, - {"name": "receive_buffer_size", "type": "field", "size": 4}, - {"name": "send_buffer_size", "type": "field", "size": 4}, - {"name": "max_message_size", "type": "field", "size": 4}, - {"name": "max_chunk_count", "type": "field", "size": 4}, - {"name": "endpoint_url", "type": "STRING"} - ], - "STRING": [ - {"name": "string_length", "type": "field", "size": 4}, - {"name": "string_value", "type": "field", "size": 14} - ] - }, - "codes" : { - "message_type": { - "HEL": "HEL_BODY" - } - } -} \ No newline at end of file diff --git a/tests/test_opcua_frame.py b/tests/test_opcua_frame.py deleted file mode 100644 index 283e78c..0000000 --- a/tests/test_opcua_frame.py +++ /dev/null @@ -1,185 +0,0 @@ -"""unittest for ``bof.opcua``. - -- OPC UA specification gathering -""" - -import unittest -from bof import opcua, BOFLibraryError, BOFProgrammingError - -@unittest.skip("Rework JSON call") -class Test01OpcuaSpec(unittest.TestCase): - """Test class for specification class building from JSON file. - TODO: Some tests are generic to BOFSpec and could be moved. - """ - def test_01_opcua_spec_instantiate_default(self): - """Test that the current `opcua.json` default file is valid.""" - try: - spec = opcua.OpcuaSpec() - except BOFLibraryError: - self.fail("Default opcua.json should not raise BOFLibraryError.") - def test_02_opcua_spec_instantiate_custom_valid_json(self): - """Test that a custom and valid opcua spec file works as expected.""" - try: - spec = opcua.OpcuaSpec() - spec.load("tests/jsons/valid.json") - except BOFLibraryError: - self.fail("Valid json spec should not raise BOFLibraryError.") - def test_03_opcua_spec_instantiate_custom_invalid_json(self): - """Test that a custom and invalid opcua spec raises exception.""" - with self.assertRaises(BOFLibraryError): - spec = opcua.OpcuaSpec() - spec.load("tests/jsons/invalid.json") - def test_04_opcua_spec_instantiate_custom_invalid_path(self): - """Test an invalid custom path raises exception.""" - with self.assertRaises(BOFLibraryError): - spec = opcua.OpcuaSpec() - spec.load("tests/jsons/unexisting.json") - def test_05_opcua_spec_property_access_valid(self): - """Test that a JSON element can be accessed as property""" - spec = opcua.OpcuaSpec() - spec.load("tests/jsons/valid.json") - frame_template = spec.frame - self.assertEqual(frame_template[0]["name"], "header") - def test_06_opcua_spec_property_access_invalid(self): - """Test that a unexisting JSON element can't be accessed as property""" - spec = opcua.OpcuaSpec() - spec.load("tests/jsons/valid.json") - with self.assertRaises(AttributeError): - frame_template = spec.unexisting - def test_07_opcua_spec_get_blocks_valid(self): - """Test that we can get block from spec as expected""" - spec = opcua.OpcuaSpec() - spec.load("tests/jsons/valid.json") - block_template = spec.get_block_template(block_name="HEADER") - self.assertEqual(block_template[0]["name"], "message_type") - def test_08_opcua_spec_get_blocks_invalid(self): - """Test that an invalid block request returns None""" - spec = opcua.OpcuaSpec() - spec.load("tests/jsons/valid.json") - block_template = spec.get_block_template(block_name="INVALID") - self.assertEqual(block_template, None) - def test_09_opcua_spec_get_item_valid(self): - """Test that we can get an item from spec as expected""" - spec = opcua.OpcuaSpec() - spec.load("tests/jsons/valid.json") - item_template = spec.get_item_template("HEL_BODY", "protocol_version") - self.assertEqual(item_template["name"], "protocol_version") - def test_10_opcua_spec_get_item_invalid(self): - """Test that an invalid item request returns None""" - spec = opcua.OpcuaSpec() - spec.load("tests/jsons/valid.json") - item_template = spec.get_item_template("INVALID", "INVALID") - self.assertEqual(item_template, None) - def test_11_get_association_str_valid(self): - """Test that a valid association is returned as expeted""" - spec = opcua.OpcuaSpec() - spec.load("tests/jsons/valid.json") - message_structure = spec.get_code_value("message_type", "HEL") - self.assertEqual(message_structure, "HEL_BODY") - def test_12_get_association_str_invalid(self): - """Test that an invalid association returns None""" - spec = opcua.OpcuaSpec() - spec.load("tests/jsons/valid.json") - message_structure = spec.get_code_value("INVALID", "INVALID") - self.assertEqual(message_structure, None) - def test_13_get_association_bytes_valid(self): - """Test that a valid association with byte id is returned as expeted""" - spec = opcua.OpcuaSpec() - spec.load("tests/jsons/valid.json") - message_structure = spec.get_code_value("message_type", b"HEL") - self.assertEqual(message_structure, "HEL_BODY") - def test_14_get_association_bytes_invalid(self): - """Test that an invalid association with byte id returns None""" - spec = opcua.OpcuaSpec() - spec.load("tests/jsons/valid.json") - message_structure = spec.get_code_value("INVALID", b"INVALID") - self.assertEqual(message_structure, None) - -class Test02OpcuaField(unittest.TestCase): - """Test class for field crafting and access. - Note that we don't test for the whole BOFField behavior, but - uniquely what we are using in OpcuaField. - TODO: Some tests are generic to BOFField and could be moved. - """ - def test_01_opcua_create_field_manual(self): - """Test that we can craft a field by hand and content is set""" - field = opcua.OpcuaField(name="protocol_version") - self.assertEqual(field.name, "protocol_version") - def test_02_opcua_create_field_template(self): - """Test that we can craft a field from a template and content is set""" - spec = opcua.OpcuaSpec() - item_template_field = spec.get_item_template("HEL_BODY", "protocol_version") - field = opcua.OpcuaField(**item_template_field) - self.assertEqual(field.name, "protocol_version") - def test_03_opcua_field_set(self): - """Test that a field value can get set as expected.""" - field = opcua.OpcuaField(name="protocol_version", size=4) - field.value = b'\x00\x00\x00\x01' - self.assertEqual(field.value, b'\x00\x00\x00\x01') - def test_04_opcua_field_set_large(self): - """Test that if we set a value that is to large, it will be cropped.""" - field = opcua.OpcuaField(name="protocol_version", size=4) - field.value = b'\x00\x00\x00\x00\x01' - self.assertEqual(field.value, b'\x00\x00\x00\x01') - def test_05_opcua_field_set_small(self): - """Test that if we set a value that is to small, it will be extended.""" - field = opcua.OpcuaField(name="protocol_version", size=4) - field.value = b'\x00\x00\x01' - self.assertEqual(field.value, b'\x00\x00\x00\x01') - -class Test03OpcuaBlock(unittest.TestCase): - """Test class for block crafting and access.""" - def test_01_opcua_create_block_empty(self): - """Tests that an empty block can be created and returns an empty list of fields""" - block = opcua.OpcuaBlock() - self.assertEqual(block.content, []) - def test_02_opcua_create_block_template(self): - """Tests that a block can be created as expected from a template""" - item_template_block = {"name": "header", "type": "HEADER"} - block = opcua.OpcuaBlock(**item_template_block) - self.assertEqual(block.content[0].name, "message_type") - def test_03_opcua_create_block_template_invalid(self): - """Tests that block creation with invalid template fail case is handled""" - with self.assertRaises(BOFProgrammingError): - item_template_block = {"name": "header", "type": "unknown"} - block = opcua.OpcuaBlock(**item_template_block) - def test_04_opcua_create_block_type(self): - """Tests that a block can be created as expected from a type name""" - block = opcua.OpcuaBlock(type="HEADER") - self.assertEqual(block.content[0].name, "message_type") - def test_05_opcua_create_block_type_invalid(self): - """Tests that block creation with invalid type name fail case is handled""" - with self.assertRaises(BOFProgrammingError): - block = opcua.OpcuaBlock(type="unknown") - def test_06_opcua_create_nested_block(self): - """Test for manual creation of nested block""" - block = opcua.OpcuaBlock(name="header", type="HEADER") - sub_block = opcua.OpcuaBlock(name="sub_block", type="STRING", parent=block) - block.append(sub_block) - self.assertEqual(block.sub_block.attributes, ['string_length', 'string_value']) - def test_07_opcua_create_dependency_block_missing(self): - """Test nested block creation with missing dependency values (neither from - defaults or parent block""" - with self.assertRaises(BOFProgrammingError): - item_template_block = {"name": "body", "type": "depends:message_type"} - block = opcua.OpcuaBlock(**{"name": "body", "type": "depends:message_type"}) - def test_08_opcua_create_block_value(self): - """Test block creation with a value to fill it""" - block = opcua.OpcuaBlock(type="STRING", value=14*b"\x01") - self.assertEqual(block.string_length.value, b'\x01\x01\x01\x01') - def test_09_opcua_create_dependency_bytes(self): - """Test nested block creation from raw bytes""" - data1 = b'HEL\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' - data2 = b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' - block = opcua.OpcuaBlock() - block.append(opcua.OpcuaBlock(value=data1, parent=block, **{"name": "header", "type": "HEADER"})) - block.append(opcua.OpcuaBlock(value=data2, parent=block, **{"name": "body", "type": "depends:message_type"})) - self.assertNotEqual(block.body, None) - def test_10_opcua_create_dependency_bytes_wrong(self): - """Test nested block creation from raw bytes with wrong dependency""" - with self.assertRaises(BOFProgrammingError): - data1 = b'XYZ\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' - data2 = b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' - block = opcua.OpcuaBlock() - block.append(opcua.OpcuaBlock(value=data1, parent=block, **{"name": "header", "type": "HEADER"})) - block.append(opcua.OpcuaBlock(value=data2, parent=block, **{"name": "body", "type": "depends:message_type"}))