diff --git a/changelogs/unreleased/8594-Lyndon-Li b/changelogs/unreleased/8594-Lyndon-Li new file mode 100644 index 0000000000..0241e34826 --- /dev/null +++ b/changelogs/unreleased/8594-Lyndon-Li @@ -0,0 +1 @@ +Data mover restore for Windows \ No newline at end of file diff --git a/config/crd/v2alpha1/bases/velero.io_datadownloads.yaml b/config/crd/v2alpha1/bases/velero.io_datadownloads.yaml index c322a23859..cc93de4d96 100644 --- a/config/crd/v2alpha1/bases/velero.io_datadownloads.yaml +++ b/config/crd/v2alpha1/bases/velero.io_datadownloads.yaml @@ -92,6 +92,13 @@ spec: DataMover specifies the data mover to be used by the backup. If DataMover is "" or "velero", the built-in data mover will be used. type: string + nodeOS: + description: NodeOS is OS of the node where the DataDonwload is processed. + enum: + - auto + - linux + - windows + type: string operationTimeout: description: |- OperationTimeout specifies the time used to wait internal operations, diff --git a/config/crd/v2alpha1/bases/velero.io_datauploads.yaml b/config/crd/v2alpha1/bases/velero.io_datauploads.yaml index 005b11e5ff..02f6cdcbbc 100644 --- a/config/crd/v2alpha1/bases/velero.io_datauploads.yaml +++ b/config/crd/v2alpha1/bases/velero.io_datauploads.yaml @@ -144,7 +144,8 @@ spec: description: DataUploadStatus is the current status of a DataUpload. properties: acceptedByNode: - description: Node is name of the node where the DataUpload is prepared. + description: AcceptedByNode is name of the node where the DataUpload + is prepared. type: string acceptedTimestamp: description: |- @@ -175,6 +176,13 @@ spec: node: description: Node is name of the node where the DataUpload is processed. type: string + nodeOS: + description: NodeOS is OS of the node where the DataUpload is processed. + enum: + - auto + - linux + - windows + type: string path: description: Path is the full path of the snapshot volume being backed up. diff --git a/config/crd/v2alpha1/crds/crds.go b/config/crd/v2alpha1/crds/crds.go index d67770b45a..d19dadca48 100644 --- a/config/crd/v2alpha1/crds/crds.go +++ b/config/crd/v2alpha1/crds/crds.go @@ -29,8 +29,8 @@ import ( ) var rawCRDs = [][]byte{ - []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xbcYKs\xe3\xb8\x11\xbe\xfbWtM\x0es\x19\xc9;\xc9V*\xa5\xdbXN\xaa\\ٙ\xa8֎\xef Ѣ\xb0\x06\x01\x04\x0f9\xca㿧\x1a )\x90\x84,K\xbb\x19\xdd\x044>|\xe8n\xf4\x03\\,\x167̈g\xb4Nh\xb5\x02f\x04\xfeӣ\xa2\x7fn\xf9\xf2'\xb7\x14\xfav\xff\xf9\xe6E(\xbe\x82up^\xb7?\xa3\xd3\xc1\xd6x\x8f[\xa1\x84\x17Zݴ\xe8\x19g\x9e\xadn\x00\x98R\xda3\x1av\xf4\x17\xa0\xd6\xca[-%\xdaE\x83j\xf9\x12*\xac\x82\x90\x1cm\x04\xef\xb7\xde\xff\xb0\xfc\xfc\xe3\xf2\x87\x1b\x00\xc5Z\\\x01\xe1q\xfd\xaa\xa4f\xdc-\xf7(\xd1\xea\xa5\xd07\xce`M\xc0\x8d\xd5\xc1\xac\xe08\x91\x16v\x9b&\xc2\xf7̳\xfb\x0e#\x0eK\xe1\xfc_gS?\t\xe7㴑\xc129\xd9;\xce8\xa1\x9a \x99\x1d\xcf\xdd\x00\xb8Z\x1b\\\xc17\xdaڰ\x1ai\xac;S\xa4\xb2\x00\xc6y\xd4\x12\x93\x1b+\x94G\xbb\xd62\xb4\xbdv\x16\xc0\xd1\xd5V\x18\x1f\xb5\x90\xd3\x02\xe7\x99\x0f\x0e\\\xa8w\xc0\x1c|\xc3\xd7\xdb\a\xb5\xb1\xba\xb1\xe8\x12-\x80_\x9cV\x1b\xe6w+X&\xf1\xa5\xd91\x87\xddlR\xe5c\x9c\xe8\x86\xfc\x81\xf8:o\x85jJ\f\x9eD\x8b\xc0\x83\x8d&\xa4s\xd7\b~'ܘ\xda+sD\xcfz\xe4'\x89\xc4y\x82s\x9e\xb5f\xca([\x9a(q\xe6\xb1Dh\xad[#\xd1#\x87\xea\xe0\xb1?\xc6Vۖ\xf9\x15\b\xe5\xff\xf8\xe3i]t\xcaZƥ\xf7Z\x8d\x15sG\xa3\x90\r'&d\xa5\x06mQ;\xda3\xf9k\x88x\x02\xb8\xcb\xd6'&\t7\x1f?K\x85\\\x0e\xf4\x16\xfc\x0e\xe1\x8e\xd5/\xc1\xc0\xa3ז5\b?\xe9:\x99\xefu\x87\x16\xa3D\x95$\xc8{A\x90\xed\xb4-\x9a\xce`\xbdL\xb2\x1dX\x8f5\xb1\xdfx\xa3\xdfܷj\x8b\xac\xe8[}\xa8YF\t\xa1U\xd9\xc1\xbe4\xf8.\xe7ʕ\xa84\xc7Lc#N\u0081\xb1\xbaF\xe7\xdepx\x02\x18\xb1\xf8v\x1c\x98\xa9&I\xec\x7fϤٱ\xcf)\xc8\xd4;l٪[\xa1\r\xaa/\x9b\x87\xe7?<\x8e\x86ፀ\xc1j\xef(R\x10}c\xb5\u05f5\x96P\xa1\x7fET\xc9\xf4\xadޣ\xa58\xd7\b\xe5\x06D\x8a\xda<\x178\xc6l\xf2\xef\x88G\xb3i\xd2b\xf4\x1e\"hs\xeb\x03\xedi\xd0z\xd1G\xe1\x0e\xfb\x98`\xb2\xd1\xc99\xfe\xb3\x18\xcd\x01\xd0\xd1\xd3*\xe0\x94i0\x1d\xab\x8b\xad\xc8;m%\xe3\t\a\x16\x8dE\x87*\xe5\x1e\x1af\nt\xf5\v\xd6~9\x81~DK0\xe0v:HN\x87ݣ\xf5`\xb1֍\x12\xff\x1a\xb0\x1dx\x1d7\x95̣\xf3\xf12Z\xc5$\xec\x99\f\xf8\x89\x946An\xd9\x01,Ҟ\x10T\x86\x17\x17\xb8)\x8f\xaf\xa4E\xa1\xb6z\x05;\xef\x8d[\xdd\xde6\xc2\xf7i\xb7\xd6m\x1b\x94\xf0\x87\xdbh\rQ\x05\xaf\xad\xbb\xe5\xb8Gy\xebD\xb3`\xb6\xde\t\x8f\xb5\x0f\x16o\x99\x11\x8bx\x10\x15S\xef\xb2忳]\xa2v\xa3mg\x8e\x98~1a^`\x1eʢt+X\a\x95\x8ex\xb4\x02\r\x91\xea~\xfe\xf3\xe3\x13\xf4L\x92\xa5\x92Q\x8e\xa23\xbd\xf4\xf6!m\n\xb5E\x9b\xd6m\xadn#&*n\xb4P>\xfe\xa9\xa5@\xe5\xc1\x85\xaa\x15\x9e\xdc\xe0\x1f\x01\x9d'\xd3Maױ4\x81\n!\x18\x8a\a|*\xf0\xa0`\xcdZ\x94k\xe6\xf0;ۊ\xac\xe2\x16d\x84wY+/\xb8\xa6\xc2I\xbd\xd9D_1\x9d0m\x1eA\x1e\r\xd6dUR,-\x13[\xd1e\x12\n\x03l$;\xd6P\xf9\xeaӯ\x98M\xa6B\xe7܍~w%\xa0\x9e\xad\xca\x02y\x97\xeb\\\x97\xa4\xe48I\xe5\xbfY~\xb4h\xb4\x13^\xdb\xc31KN]\xe1\xa4U\xe8W3U\xa3\xbc\xe6x\xeb\xb8\x12\x84\xe2\xa4s\x1c\\\x99\x82PB\x8dD\xb5j4]\xae\x91)\xe0\xc1\x93\f\xf9\xb6C_>\xa8*f5\xa1\xe0XSB^;N\x8f[i-\x91M\xb5H^\xf8\x95\xd2\xc2Z\xab\xadh\xe6\a\xcf\xcb\xdfS.rF\xa7\x05\x87Ͷ\xa4S\x90w\x12\x93E\xccP\x8b\xdeu)\xb4oE\x13\xec)\xfbo\x05J>\x8b?'oR\x7f\xe0\xb8\xcb56\x1e\xa8\xf7\xb7\xab\xcbjY\xea\xf5:F(\x17\xeb\xdd\xcc5\xe7$\x01\x1e\xb6\x19\xa2p\xf0\xe1\x03h\v\x1fRO\xf4\xe1SZ\x1d\x84\xf4\v1\xca\xff\xafB\xca~\x97\x8b\xbc{H\xf9Tu\xe9\xe0\xaf\xd1\xc1\xdf&\x18\x13Ux\xaa\x11\xe3\xf1\xbd\x86W&\xb2\xb4;\xec\xee>\x15p+\xdcR\x8c\xb6\xe8\x83UtC\xd0Z\nZ.B\xea0+\x03\xde<\xa9S̸\x9d\xf6\x0f\xf7g\xce\xf88\b\xf6\xa1\xe8\xe1\xbe\x0fD\xcf\xd1\x10C<\xea$\xc1\xeb\"\xfd\xbe\xb0\xe21\xd3]\xc66\xa6ס\t\xbd\xc6,\x8fc\x88\xfe0ڊF\x90\xf2\xd50s\f\x9a{\xead\xa3(\x1d\x119\x04s\x82;P\x84\xa2|^!p\xb1ݢ\xa5\xa4\x1d3z\xdax\xf3\xbc\xfe\xe8\xb2M\xc46\xffC\xc1\xb0e\xc6 \xa7\xf6\x81\x8c\xdb\xe9\xea\"-yf\x1b\xf4ϑ\xf4\x19\x15=e\xa2\xbd*(\xfbS\xafו\x97\xd1Y\xa3\x18l\x9eׅb\x90~\x9b\xe79\xc3ө\x12\xba\xbe\xe0\x84\x11g,g\xd6\xea\xf8\f\x18E\x887#-\x80ٿc\xe7\xcds)\xf1\x0e\xea\x00\xbfc\x9e$\xba>\x0e\xaaC\x11\x13\xfa+ҙ\xf3:\xbe\xf5\xbb\b\xaf\xdfd\xbc\x9eR>\xc1\xb7:\xfcjʔׅE>g\xbdx\xc3r\v0\xfb\xe2`\xfd\xfe\xecU\xdeyQ.\xd1&2\xd3\xd0?\x99>\xc6\xcb\xe9\xc48\xaeLf\xf3+\xf9\xaeZ6v\xda\xef\xadf\xd3\xfbYg\xf6:\xd8\x18t\xbaW5j\x10\xaf\xaagY]\xa3\xf1\xc8\xef\x0e\xd4ޟ\x89#$B\x04\xd4\xdb\xef\f\x7f7\xc7W\x064\xecҢ\xb3\xa74\xbc\x85\\\x93\x00\xbeLAbCly\x96\x96\xe7tS\xb5r\x9a4\xc0\x13\xb5\x12\xb1\xa1\xfb\x9821-\x8b\xf9\x9d\x8a\xb6٦3\x84\xfe}\x8d:\xb6\x05\xad\x9fI\xa8 %\xab$\xae\xc0\xdbp\xaaz-\x17\xeb\xe9i1\x7fE\xba\xaar\x9f\xc3\xccudžw\x93\xf8\xbe\xd5?j\x96Tv\xc4\x1b\x14\x96\xe0\x90\x03\xeeQ\x01\xf5cLH\xe4=f\xa1\x84=\xa7\xf9\x02i\xf7]\x95ߢs\xac9w\x81\xbe&\xa9\xf4\xd4\xd0-\x01VQ\xdd8mg>\xba\xeen_tw\xd4ov\x89\x8b\x8f\x85\x17q\x89\xed\xd7\x192\x1b\x92)Ŵ\x81\xda\xe9\xa0F?T\xa1-e\x9eo\xf8Z\x18\xed\xefgaj\xd3]\xfa\xc2\xd4\xec+E>\x99\xfa\xdcRb\xec犘\xc3g\x80\xc2\xdc_\xe2e\xb8H\xd3\x1d\xbfk\xae\xfb\xd0-\xef\xb4\xecox|\xbeW\xa1\xadВ\x19\xe2\a\x82\xde\x1eC\xdd\xcf\x14ϭV*\xfe\x06\x84\xa1\x17\x88PKx\xdaQi\x92Z\xfc\xbe;\xe2\xc2\x19\xc9\x0e\xc3a\xf2\n\xb5\x00~\xbc5\xb3\x17\xdcK\x8b\xd4\xe1sJ\xb9\xf2*}\x13\x19\xff\xe6_7&\xf3\xc3g\x92\xff\xcf\x0eo4\xf8\xe3\xcfVW\xb5R#\x84s\xa9\xa0\xfb\x8cvy\x04\x1fo\xf3=\x83wQ{\xb3\xc1Ȝg\xd8݃\\>\x12\xaa\xe1\x95z\x05\xff\xfe\xef\xcd\xff\x02\x00\x00\xff\xff\x84s\xba\x82\x91\x1e\x00\x00"), - []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xbcYIs\xe3\xb8\x15\xbe\xfbW\xbc\xea\x1c\xe6b\xc9\xd3\xc9T*\xa5[[N\xaa\\\x99vT-\xc7w\x88|\"1\x06\x01\x06\x8b\x14g\xf9\xef\xa9\a\x80\x14HBk\xa6\x87\a\x97\x85\xe5\xe1m\xf8ނ\xd9lv\xc7Z\xfe\x86\xdap%\x17\xc0Z\x8e\xff\xb4(闙\xbf\xff\xc9̹z\xd8}\xbe{\xe7\xb2\\\xc0\xd2\x19\xab\x9aoh\x94\xd3\x05>\xe1\x96Kn\xb9\x92w\rZV2\xcb\x16w\x00LJe\x19\r\x1b\xfa\tP(i\xb5\x12\x02\xf5\xacB9\x7fw\x1b\xdc8.JԞxw\xf4\xee\xc7\xf9\xe7\x9f\xe6?\xde\x01H\xd6\xe0\x02\x88\x9ek\x85b\xa5\x99\xefP\xa0Vs\xae\xeeL\x8b\x05\x91\xad\xb4r\xed\x02\x0e\x13a[<2\xb0\xfb\xc4,\xfb\xbb\xa7\xe0\a\x057\xf6\xaf\xa3\x89\x9f\xb9\xb1~\xb2\x15N318Տ\x1b.+'\x98Ng\xee\x00L\xa1Z\\\xc0\v\x1dٲ\x02i,J\xe2Y\x98\x01+K\xaf\x1b&V\x9aK\x8bz\xa9\x84k:\x9d̠DSh\xdeZ/\xfb\x81!0\x96Yg\xc0\xb8\xa2\x06f\xe0\x05\xf7\x0f\xcfr\xa5U\xa5\xd1\x04\x96\x00~1J\xae\x98\xad\x170\x0f\xcb\xe7m\xcd\f\xc6٠\xbe\xb5\x9f\x88C\xf6\x83\xb85VsY\xe5\xce\x7f\xe5\rB\xe9\xb47\x1b\xc9\\ ؚ\x9b\x94\xb1=3Ĝ\xb6X\x1ee\xc3\xcf\x131cYӎ\xf9I\xb6\x06\x86Jf1\xc7\xceR5\xad@\x8b%l>,vBl\x95n\x98]\x00\x97\xf6\x8f?\x1d\xd7DT\xd5\xdco}Rr\xa8\x96G\x1a\x85d8pB\x16\xaaPgu\xa3,\x13\xff\x0f#\x96\b<&\xfb\x03'\x81n:~\x96\x15r7P[\xb05\xc2#+\xde]\vk\xab4\xab\x10~VE0\u07beF\x1d\x8d\xb7\tKL\xad\x9c(a\xd3I\f`\xac\xd2Y+\xb6X\xccîH\xb7#;2\xe5\xf0\xcc_\xd9\xc9\n\x8d,\xebd\x1d\xca\xcc\xfd\n\xaed\xdeӾTx\x91\x97\xa5ڔ\xaa\xc4^u\x98r\xc4\r\xb4Z\x15h\xcc\t\xbf\xa7\xed\x03\x1e^\x0e\x03\x13\xb5\x84\x15\xbb\xdf3\xd1\xd6\xecs@\x99\xa2Ɔ-\xe2\x0eբ\xfc\xb2z~\xfb\xc3z0\fG1\x83\x15\xd6\x10X\x10\xeb\xadVV\x15J\xc0\x06\xed\x1eQz܂F\xedP\x13\xc8U\\\x1a`\xb2\xeciB\xba\xe0\x00\xd5\xe4\xe4\x9e\x1e͆\xc9\xe8N\xaaE\x9d\x9a\x1d\xe8\xc8\x16\xb5\xe5\x1d\xfa\x86/\t+\xc9\xe8H\x88\xff\xcc\x06s\x00$w\xd8\x05%\xc5\x17\fREl\xc52\xaa*؍\x1b\xd0\xd8j4(Cġa&Am~\xc1\xc2\xceG\xa4ר\x89Lw\x1f\n%w\xa8-h,T%\xf9\xbfz\xda\x06\xac\xf2\x87\nf\xd1X\x7f!\xb5d\x02vL8\xbc\x1fi\x8f\xbe\x86}\x80F:\x13\x9cL\xe8\xf9\rf\xcc\xc7W\xa5\x11\xb8ܪ\x05\xd4ֶf\xf1\xf0Pq\xdb\x05\xdbB5\x8d\x93\xdc~\xe0Λ\xf2w:\x86g38v\xe2\x85\xe1\xf3\x81\xf2\n\xf3P\xfc\xa4+\xc1\"\xa9 \xe2\xc1\n4D\xaa\xfb\xf6\xe7\xf5+t\x9c\x04K\x05\xa3\x1c\x96N\xf4\xd2ه\xb4\xc9\xe5\x16uطժ\xf14Q\x96\xad\xe2\xd2\xfa\x1f\x85\xe0(-\x18\xb7i\xb8%7\xf8\x87Cc\xc9tc\xb2K\x9f\x90\xc0\x06\xc1\xb5\x04\x05\xe5x\xc1\xb3\x84%kP,\x99\xc1\xdf\xd8Vd\x153##\\d\xad4\xcd\x1a/\x0e\xeaM&\xbaL\xe9\x88i\x0f\xf0\xb1n\xb1 \x9b\x92Zi\x13\xdf\xf2\x18K\b\x03X\xb2r\xa8\x9d\xfc\xb5\xa7/\x1bBƋι\x1a}\x8f9B\x1d\xaf2\xc1\xef.\xd4\xc5\xc8$\x86\x91)\xfd\x0e \x1f\xf7hl\x95\xe1V\xe9\x0f\"\x1cB\xe3\xd8\r\x8eZ\x84\xbe\x82\xc9\x02\xc5-\xe2-\xfdN\xe0\xb2$\x8dc\xef\xc6\x04@\x81\xaagT\xc9J\xd1\xc5J\f\x01ϖV\x90W\x1b\xb4y1e&\x94q\t\x87l\x12Ҭq,\xeaF)\x81l\xac\xc1\xc2\xf0\xb5d\xad\xa9\x95=#\xf0\xf3\x16\xba\x95\xaf\x1f-\xd2\xe1\xcb\xf5\xf3=\xfd\xe9\xc6Ƀv\xbc\x8c\x10O\xb7\x8c\xf2\xaa\xbc٢\x9d\x97\xebg0q\xfb\xd4H\xd2\t\xc16\x02\x17`\xb5\x9b\nv\xdca\xe9\xeb\xc8.\x053\xd9\x05#\x01\xd7\xe9\xfa\x9cOv\x04\xa1\xf0+l\xcdr\x86\xf2\x1a\xa7\bG\xe5A\xb2\x89\xf7\x89\x10칭\xb3;O8%\xc44\x8fUx\xb1@\xc9\xf2\xac<\xf1r\x05q\xd4\xf6\x840\xab\xb7\xa5\x97\xf7\x9cd\x84\xed\xb7H\x16H\x1e\xf7ĉlo\x83\r9\xe9F\\\x1e\x13Nѕ#\xe4\xc0\x12\\{=\xeftù\xc6r\xca\xf3l`\xaf\xcc\xf4P\xe8#\xd7v\x12\x06 fx_)\x87[*\xb9\xe5\xd5\xf4\xec\xb4X=uGN\x8a6\t/ɑ\xa4q\x8a&\xc4\xc9̧\x93\xb3.\xd4P\"\xb6\xe5\x95\xd3Ǯ\xfe\x96\xa3('\xd9\xc2\xd9\xdb~F\x1f\x9e\x89[@\xbb\x97\xac\v\x96\x11\xbf\x924:x\x893\xbe\x80Mb\xcdT\x06 \x9c}\x02\xa5\xe1Shl|\xba\x0f\xbb\x1d\x17v\xc6\a\xb9\xfc\x9e\vѝrU\xb8\xea\xf3w\xaa\x9e\x94;\x87\xe3Y\x1d\xfcmDc\xa4\nK\x95\x9e\x17\xdf*\xd83\x9e\xe4\xd0\xfd\xe9\xe6>Cw\x83[J\xb84Z\xa7%\x85<Ԛr\x10\xe3I*\x97\xc1\xfc\x13\x92\x9a$\xfe\x9c\x91r\x1c\xaa\xbc\x14\xf4\xff\x18\xcbS\x00\xc8\b\x90\xb3\xf1)\x0e}~\xdcw\x91n1\xc5zH\xa2c^i^qR\xb8\xecg\x0e\x99Oĺ\xd8\"\xf0H\xe6\xa18\xeb\x9f=Z\x1aB\xcb\x039\xba\xce\xe1pB{&K\x1f\x9c\xfb\xf92^\xbd\xcc\xc5=\xab\x90\xd5\xdb\xf2\x9c\xbd\xfa\x833PN\xc3\xfb\x9a\x17\xf5\xd0t|\n\xaa\x00\x96\xbd\xa3Ot\xaf`3\x8f\xe1\xb3|\xda;Z3\xbe}\xa3\xe9\xd4e\xc7SCCggWoˋJ\x03ߵ\xb8\xac8\b\xedȨ\xe5\xc2i\xedˮ0J\xd5\xf6\r\xe5\x01+\nl-\x96\x8f\x1f/\xaa<\xe7\xf4\xb4\x84\x8e\x97\x97wk\xb0e\xd7\xe6\xf0\x1dK}G閫\xf8eL\xc4\xf7\x16t\x99\x80\xe2\x94\xdd\x00(Ǚ\x06x%'\xf6\xb5\xf1\x0f\x01\ai\x9bGW\xba\x82\x93C'\x14\xbav%\x15\xbf3\xda\x7f[$\xcd\xd7>\xa1S\x9b\xf6\xe2n*\x84\xa6d\xa6\xbac]\xc5曄]\x8b8\xa7\xb1\x03\xb9^_\x81\x1a\x96\x80;\x94@\xb5-\xe3\x82\xe2\xb3'\x99\x01\xa9\xd3Tb\xa0\n\xef\x01]ӣk\x90e\xbbO\xe7-\x99Q\xc2\x14\xb1\xbe\xa71\xfb4\xf1\x1b\x1a'2\x89\xc1wL\x13Ñ\xa1\xfc6\xd94\xf1t}\xc8\f0ЁHD\x89c\xc0t\xb1\x92\xb2\xb9c\x83ư\xea\x1cj}\r\xabB\xab,n\x01\xb6\xa1Ti\xc8\xda\x0f&\x82\xe9Up%\xbf\vn\xc6.\xf7U\x9c\xb4\xcc\xd6g8Y1[w\x01d\xeb\x84\xf0{&\x89U\xccI6H\xb7\xe9\xd7ʯ|\x83\xe3\x1c{\xb4&\x17\xe0\xf0\x12GB\xe9\x9a\\-\xf7\x82\xfb\xcch\a֙\xa9U\x8c\x00\x99\xa9\xc9\xfb_:\x19zH\xb9\x8b\xd6\xcdei\xf6Ol\x99\xb9\xbfxh\xbcJϑ\xbf[\xb0\xbf\xefF\xd5Jtp\xef\x9fƤk6\xa8\xc9\b\xfe\xf1mT\xa8S\u0099X,C8\xd9\xdfg\xb9\x9e\xd2\x1c^kn\xba\xfeYW\xa7\x94ܴ\x82}\xf4\xb2\x9c\x03\x9c\xfe2\x8f\xdfE\xa6Nr\xba\xf1\xd4?T\xe6\xfb\x18\xb9\xd7\xc6\xe17}7\x1c\xcd\xf7\x0f\x90\xdf\xe7\x84\x13h\xd9]\xef\xe7\xa7\v\v\xb0\xe7\xa7\xee*\xf2\x12\xa5\xa5\x9a\xf2\xf0\x16uH\xe5}o3\xa7\xcbqO\xf7\xba\xeac\xf0|}S56\xa0p&\x87\x89\xaf\xe9\xb9LaM`@\x10\xe4_?\x96\xe3\xf7\xce\xfb\xfe\xf9\x94\xd9\xf8\x04S\xd4LV\x98+q\x94\xa4\xc0\xe8\x03\xeb\xf5I\xc9P\xa0\xdf2\x1f\xc9z\xd5d\xd0s^&\xb4c\x13-\x1dq\x9b\xfeMl\x01\xff\xfe\xef\xdd\xff\x02\x00\x00\xff\xff*b\xfd\xb1\xf5\"\x00\x00"), + []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xbcYKs#\xb9\r\xbe\xfbW\xa0&\x87\xb9\x8c\xe4\x9dd+\x95\xd2m,'U\xae\xecxT+\xc7wv\x13jq\xcd&\x19>\xa4U\x1e\xff}\vdw\xab\x1f\x94di\x1f\xba\x89\x04\xc1\x8f\x00\b|`\xcff\xb3;f\xc4+Z'\xb4Z\x003\x02\x7f\xf6\xa8蟛\xbf\xfd\xcdͅ\xbe\xdf}\xbe{\x13\x8a/`\x19\x9c\xd7\xf5\x8f\xe8t\xb0%>\xe2F(\xe1\x85Vw5zƙg\x8b;\x00\xa6\x94\xf6\x8c\x86\x1d\xfd\x05(\xb5\xf2VK\x89vV\xa1\x9a\xbf\x85\x02\x8b $G\x1b\x95\xb7[ﾛ\x7f\xfe~\xfe\xdd\x1d\x80b5.\x80\xf4q\xbdWR3\xee\xe6;\x94h\xf5\\\xe8;g\xb0$ŕ\xd5\xc1,\xe08\x91\x166\x9b&\xc0\x8f̳\xc7FG\x1c\x96\xc2\xf9\x7fN\xa6~\x10\xce\xc7i#\x83er\xb4w\x9cqBUA2;\x9c\xbb\x03p\xa56\xb8\x80g\xdaڰ\x12i\xac9S\x842\x03\xc6y\xb4\x12\x93++\x94G\xbb\xd42ԭuf\xc0ѕV\x18\x1f\xadЇ\x05\xce3\x1f\x1c\xb8Pn\x819x\xc6\xfd\xfd\x93ZY]Yt\t\x16\xc0ON\xab\x15\xf3\xdb\x05̓\xf8\xdcl\x99\xc3f6\x99r\x1d'\x9a!\x7f \xbc\xce[\xa1\xaa\x1c\x82\x17Q#\xf0`\xa3\v\xe9\xdc%\x82\xdf\n7\x84\xb6g\x8e\xe0Y\x8f\xfc$\x908O\xea\x9cg\xb5\x19#\xea-M\x908\xf3\x98\x03\xb4Ե\x91\xe8\x91Cq\xf0\xd8\x1ec\xa3m\xcd\xfc\x02\x84\xf2\x7f\xfd\xfe\xb4-\x1ac\xcd\xe3\xd2G\xad\x86\x86y\xa0Q\xe8\r'$\xe4\xa5\nm\xd6:\xda3\xf9k\x80xR\xf0\xd0[\x9f\x90$\xbd\xfd\xf1\x8bP(\xe4@o\xc0o\x11\x1eX\xf9\x16\f\xac\xbd\xb6\xacB\xf8A\x97\xc9}\xfb-Z\x8c\x12E\x92\xa0\xe8\x05A\xbe\xd36\xeb:\x83\xe5<\xc96\xcaZ]#\xff\r7\xfa\xcdc\xab\xb4Ȳ\xb1զ\x9ay\x94\x10Z\xe5\x03\xecK\x85\xef\n\xae\xbe\x11\x95\xe6س\xd8\x00\x93p`\xac.ѹ3\x01O\n\x06(\x9e\x8f\x03\x13\xd3$\x89ݟ\x994[\xf69%\x99r\x8b5[4+\xb4A\xf5e\xf5\xf4\xfa\x97\xf5`\x18\xce$\fVzG\x99\x82\xe0\x1b\xab\xbd.\xb5\x84\x02\xfd\x1eQ%\xd7\xd7z\x87\x96\xf2\\%\x94\xeb4R\xd6\xe6}\x81cΦ\xf8\x8e\xfah6MZ\x8c\xd1C\x00m\xdf\xfb@{\x1a\xb4^\xb4Y\xb8\xd1},0\xbd\xd1\xd19\xfe7\x1b\xcc\x01\xd0\xd1\xd3*\xe0Ti0\x1d\xabɭ\xc8\x1bk%\xe7\t\a\x16\x8dE\x87*\xd5\x1e\x1af\nt\xf1\x13\x96~>R\xbdFKj\xc0mu\x90\x9c\x0e\xbbC\xeb\xc1b\xa9+%\xfe\xd3\xe9v\xe0u\xdcT2\x8f\xce\xc7\xcbh\x15\x93\xb0c2\xe0'2\xdaHs\xcd\x0e`\x91\xf6\x84\xa0z\xfa\xe2\x027\xc6\xf1\x95\xac(\xd4F/`\xeb\xbdq\x8b\xfb\xfbJ\xf8\xb6얺\xae\x83\x12\xfep\x1f\xbd!\x8a\xe0\xb5u\xf7\x1cw(\uf768f̖[\xe1\xb1\xf4\xc1\xe2=3b\x16\x0f\xa2b\xe9\x9d\xd7\xfcO\xb6)\xd4n\xb0\xed$\x10\xd3/\x16\xcc+\xdcCU\x94n\x05kT\xa5#\x1e\xbd@Cd\xba\x1f\xff\xbe~\x81\x16I\xf2Tr\xcaQtb\x97\xd6?dM\xa16hӺ\x8d\xd5uԉ\x8a\x1b-\x94\x8f\x7fJ)Pyp\xa1\xa8\x85\xa70\xf8w@\xe7\xc9uc\xb5\xcbHM\xa0@\b\x86\xf2\x01\x1f\v<)X\xb2\x1a\xe5\x929\xfc\x83}E^q3r»\xbc\xd5'\\c\xe1d\xde\xdeD˘N\xb8\xb6\x9fA\xd6\x06K\xf2*\x19\x96\x96\x89\x8dh*\t\xa5\x016\x90\x1dZ(\x7f\xf5闭&c\xa1K\xe1F\xbf\x87\x9c\xa2\x16\xad\xea%\xf2\xa6ֹ\xa6H\xc9a\x91\xea\xff&\xf5Ѣ\xd1Nxm\x0f\xc7*9\x0e\x85\x93^\xa1_\xc9T\x89\xf2\x96\xe3-\xe3J\x10\x8a\x93ͱ\veJBIk\x04\xaaU\xa5\xe9r\r\\\x01O\x9ed(\xb6\x1d\xfa\xfcAU\xb6\xaa\t\x05GN\t}\xee8>n\xa1\xb5D6\xb6\"E\xe1W*\vK\xad6\xa2\x9a\x1e\xbcO\x7fO\x85\xc8\x05\x9bf\x02\xb6\xb7%\x9d\x82\xa2\x93\x90\xccb\x85\x9a\xb5\xa1K\xa9}#\xaa`O\xf9\x7f#P\xf2I\xfe9y\x93\xda\x03\xc7]n\xf1q\a\xbd\xbd]MU\xeb\x95^\xafc\x86r\x91\xef\xf6Bs\n\x12\xe0i\xd3\xd3(\x1c|\xf8\x00\xda\u0087\xd4\x13}\xf8\x94V\a!\xfdL\f\xea\xff^H\xd9\xeerUt\x13\xc3\xf9\xb6\xbep\xf2\xe7(Dx\xbe\xad\xcfr+\xb5\x9fp\xab)\x1aT\xa1\x9en8\x03\x16\xbc\xce\fK\xa1\xc2ϙ\xf1\xbdP\\\xef\xdd5\x87\xed\xf8\rQL\x1d\xfc-\x0e\xff6\xd21\xf2\xbb'B\x1c}\xed5\xec\x99\xe8q\x8cnw\xf7)\xa3\xb7\xc0\r\x15$\x8b>XE\xe9\x00\xad\xa5\f\xed\xa2J\x1d&\x9c\xe7\xecI\x9db\xc6m\xb5\x7fz\xbcp\xc6u'\xd8\xe6ݧ\xc7\xd6ů1\xea\xba\xe4\xdbHB\xc6K\x04\xbfe\x91<\x96\xf5\xeb\xd0F.\xd1uܷ\xb8e=T\xd1\x1eF[Q\t2\xbe\xeaf\x8e1\xbb\xa3\xb6=\x8a\xd2\x11\x91C0'\xb0\x03\xa5c\"/\x05\x02\x17\x9b\rZb(\x91\xbe\xa4\x8dW\xafˏ\xae\xb7\x89\xd8\xf4\xffP毙1ȩW\"\xe76\xb6\xba\xcaJ\x9e\xd9\n\xfdk\x04}\xc1D/=\xd1\xd6\x14Du\xa8\xb1m\xb8t\f\xd6(\x06\xab\xd7e\x86\xf9\xd2o\xf5:Ex\x9a\x17@\xd3\x04\x9dp\xe2\x04\xe5\xc4[\r\x9eNGV\xc5ٲ\x02`v\xef\xd8y\xf5\x9ac\x19\x9d9\xc0o\x99'\x89\xa6i\x85\xe2\x90\xd5\t\xed\x15i\xdcy\x1b\xde\xf2]\x80\x97g\x11/ǐO\xe0-\x0e\xbf\x1a2\x91\x18a\x91\xe7R\xf8i\xcf\xcd\xc0첃\xe5\xfbKu~\xe7Y\x9e\x8f\x8edƩ\x7f4}̗\xe3\x89a^\x19\xcd\xf6\xaf仈{|Vx/uO\x8f\x85\x8d\xdb\xcb`c\xd2i\x9e\x10\xa9\x1b\xbe\x89\xbc\xb3\xb2D\xe3\x91?\x1c\xa8\xaa\xbf\xa3\xf0\x13\x00u\xfeQ\xe5_\xe6X\xf6Ѱk\x19v\v\xa9{\xf8\xb9\xa5\x00|\x19+\x89ݿ彲<\x85\x9b\xa8\xd9i\xd0\x00/\xd47\xc5\xee\xf5c\xaaĴ,\xd6wb\xa8\x93M'\x1a\xda\xc7DjOg\xb4~\"\xa1\x82\x94\xac\x90\xb8\x00o\xc3)\xaa\x9e\xefL\xd2;j\xff\xc9\xec\xa66e\xaafj;\xd6=\x12\xc5Ǽ\xf6\x057g\xb2\xa3\xbe\xce`I\x1dr\xc0\x1d*\xa0\xe6\x93\t\x89\xbcՙ\xe1\xeb\x97,\x9f\x01=\xa5\x82\xbf\xa7\xf1kt\x8eU\x97.\xd0\xd7$\x95\xdeU\x9a%\xc0\n\xe2\x8d\xe3\xde\xed\xa3k\xee\xf6\xd5\xfc\xfd\xb7\xb9\xc4ٗѫ\xb0\xc4^\xf3\x02\x98\x15\xc9\xe4rZ\a\xedtR\x833\xcd\xc33\xee3\xa3\xed\xfd\xccL\xad\x9aK\x9f\x99\x9a|\x92\xe9O\xa6\xa6>W\x18۹\xac\xce\xee\x9bGf\xee\x1f\xf12\\e\xe9\x06\xdf-\u05fd{\x1a\xd8j\xd9\xde\xf0\xf8\xadB\x85\xba@Kn\x88_CZ\x7ft\xbc\x9f)\xde\xf7Z\x8e\xfcu\x1a\xba^ \xaa\x9a\xc3˖\xa8Iz\xcfh\xbb#.\x9c\x91\xec\xd0\x1d\xa6\xcfP3ʏ\xb7f\xf2\\}-I\xed\xbe\x1d\xe5\x99W\xee\x03\xd0\xf07\xfd\x943\x9a\xef\xbe\t\xfd>;\x9cy\xcd\x18~\xa3\xbb\xa9\x95\x1ah\xb8T\n\x9ao\x86\xd7g\xf0\xe16\x7fd\xf2\xceZo2\x18\x91\xf3\x9e\xee\xe6\xf5\xb1?\x12\x8a\xeeI~\x01\xff\xfd\xff\xdd/\x01\x00\x00\xff\xff5\x17GR~\x1f\x00\x00"), + []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xbcZIs\xe3\xb8\x15\xbe\xfbW\xbc\xea\x1c\xe6b\xc9\xd3\xc9T*\xa5[[N\xaa\\\x99v\xbbZ\x8e\xef\x10\xf9Db\f\x02\f\x16i\x9c\xe5\xbfO=\x00\xa4@\x12Zg\xbay\xe8jayx\x1b\xbe\xb7\xc0\xb3\xd9솵\xfc\x15\xb5\xe1J.\x80\xb5\x1c\x7f\xb5(闙\xbf\xfd\xcd̹\xba\xdb~\xbcy\xe3\xb2\\\xc0\xd2\x19\xab\x9a\xafh\x94\xd3\x05>\xe0\x86Kn\xb9\x927\rZV2\xcb\x167\x00LJe\x19\r\x1b\xfa\tP(i\xb5\x12\x02\xf5\xacB9\x7fsk\\;.JԞxw\xf4\xf6\xc7\xf9ǟ\xe6?\xde\x00H\xd6\xe0\x02\x88\x9ek\x85b\xa5\x99oQ\xa0Vs\xaenL\x8b\x05\x91\xad\xb4r\xed\x02\xf6\x13a[<2\xb0\xfb\xc0,\xfb\x97\xa7\xe0\a\x057\xf6\x9f\xa3\x89\x9f\xb9\xb1~\xb2\x15N318Տ\x1b.+'\x98Ngn\x00L\xa1Z\\\xc0\x13\x1dٲ\x02i,J\xe2Y\x98\x01+K\xaf\x1b&\x9e5\x97\x16\xf5R\t\xd7t:\x99A\x89\xa6м\xb5^\xf6=C`,\xb3\u0380qE\r\xcc\xc0\x13\xee\xee\x1e\xe5\xb3V\x95F\x13X\x02\xf8\xc5(\xf9\xccl\xbd\x80yX>okf0\xce\x06\xf5\xad\xfcD\x1c\xb2\xefĭ\xb1\x9a\xcb*w\xfe\vo\x10J\xa7\xbd\xd9H\xe6\x02\xc1\xd6ܤ\x8c\xed\x98!\xe6\xb4\xc5\xf2 \x1b~\x9e\x88\x19˚v\xccO\xb250T2\x8b9v\x96\xaai\x05Z,a\xfdn\xb1\x13b\xa3t\xc3\xec\x02\xb8\xb4\x7f\xfd\xe9\xb0&\xa2\xaa\xe6~냒C\xb5\xdc\xd3($Á\x13\xb2P\x85:\xab\x1be\x99\xf8=\x8cX\"p\x9f\xec\x0f\x9c\x04\xba\xe9\xf8IV\xc8\xdd@m\xc0\xd6\b\xf7\xacxs-\xac\xacҬB\xf8Y\x15\xc1x\xbb\x1au4\xde:,1\xb5r\xa2\x84u'1\x80\xb1Jg\xad\xd8b1\x0f\xbb\"ݎ\xecȔ\xc33\xff`'+4\xb2\xac\x93u(3\xf7+\xb8\x92yO\xfbT\xe1Y^\x96jS\xaa\x12{\xd5a\xca\x117\xd0jU\xa01G\xfc\x9e\xb6\x0fxx\xda\x0fL\xd4\x12Vl\xff\xccD[\xb3\x8f\x01e\x8a\x1a\x1b\xb6\x88;T\x8b\xf2\xd3\xf3\xe3\xeb_V\x83a8\x88\x19\xac\xb0\x86\xc0\x82Xo\xb5\xb2\xaaP\x02\xd6hw\x88\xd2\xe3\x164j\x8b\x9a@\xae\xe2\xd2\x00\x93eO\x13\xd2\x05{\xa8&'\xf7\xf4h6LFwR-\xea\xd4\xec@G\xb6\xa8-\xef\xd07|IXIFGB\xfco6\x98\x03 \xb9\xc3.()\xbe`\x90*b+\x96QU\xc1n܀\xc6V\xa3A\x19\"\x0e\r3\tj\xfd\v\x16v>\"\xbdBMd\xba\xfbP(\xb9EmAc\xa1*\xc9\xff\xd3\xd36`\x95?T0\x8b\xc6\xfa\v\xa9%\x13\xb0e\xc2\xe1\xedH{\xf45\xec\x1d4ҙ\xe0dB\xcfo0c>>+\x8d\xc0\xe5F-\xa0\xb6\xb65\x8b\xbb\xbb\x8a\xdb.\xd8\x16\xaai\x9c\xe4\xf6\xfd\xce\x1b\x83\xaf\x9dU\xdaܕ\xb8Eqgx5c\xba\xa8\xb9\xc5\xc2:\x8dw\xac\xe53/\x88\xf4\x01wޔ\x7f\xd21<\x9b\xc1\xb1\x13/\f\x9f\x0f\x94\x17\x98\x87\xe2']\t\x16I\x05\x11\xf7V\xa0!R\xdd\u05ff\xaf^\xa0\xe3$X*\x18e\xbft\xa2\x97\xce>\xa4M.7\xa8þ\x8dV\x8d\xa7\x89\xb2l\x15\x97\xd6\xff(\x04Gi\xc1\xb8u\xc3-\xb9\xc1\xbf\x1d\x1aK\xa6\x1b\x93]\xfa\x84\x04\xd6\b\xae%((\xc7\v\x1e%,Y\x83b\xc9\f~g[\x91Ǔ\x8cp\x96\xb5\xd24k\xbc8\xa87\x99\xe82\xa5\x03\xa6\xdd\xc3ǪłlJj\xa5M|\xc3c,!\f`\xc9ʡv\xf2מ\xbel\b\x19/:\xe5j\xf4\xdd\xe7\bu\xbc\xca\x04\xbf\xbbP\x17#\x93\x18F\xa6\xf4ۃ|ܣ\xb1U\x86[\xa5߉p\b\x8dc78h\x11\xfa\n&\v\x14\u05c8\xb7\xf4;\x81˒4\x8e\xbd\x1b\x13\x00\x05\xaa\x9eQ%+E\x17+1\x04pm'a\x00b\x86\xf7\x99r\xb8\xa5\x92\x1b^M\xcfN\x8b\xd5cw\xe4\xa8h\x93\xf0\x92\x1cI\x1a\xa7hB\x9c\xcc|:9\xebB\r%b\x1b^9}\xe8\xeao8\x8ar\x92-\x9c\xbc\xed'\xf4ᙸ\x06\xb4{ɺ`\x19\xf1+I\xa3\x83\x978\xe3\v\xd8$\xd6Le\x00\xc2\xc9=En\xe0\xc3\aP\x1a>\x84\xc6Ƈ۰\xdbqag|\x90\xcb\xef\xb8\x10\xdd)\x17\x85\xab>\x7f\xa7\xeaI\xb9S8\x9e\xd5\xc1\x97\x11\x8d\x91*,Uz^|\xab`\xc7x\x92C\xf7\xa7\x9b\xdb\f\xdd5n(\xe1\xd2h\x9d\x96\x14\xf2Pk\xcaA\x8c'\xa9\\\x06\xf3\x8fHj\x92\xf8sB\xcaq\xa8\xf2R\xd0\xff\xc7X\x9e\x02@F\x80\x9c\x8d\x8fq\xe8\xf3㾋t\x8d)VC\x12\x1d\xf3J\xf3\x8a\x93\xc2e?\xb3\xcf|\"\xd6\xc5\x16\x81G2\x0f\xc5Y\xff\xec\xd1\xd2\x10Z\xee\xc9\xd1u\x0e\x87\x13\xda3Y\xfa\xe0\xdcϗ\xf1\xeae.\xeeI\x85<\xbf.O٫?8\x03\xe54\xbc\xabyQ\x0fMǧ\xa0\n`\xd9\x1b\xfaD\xf7\x026\xf3\x18>˧\xbd\xa35\xe3\xdb7\x9aN]v<54tv\xf6\xf9uyVi\xe0\xbb\x16\xe7\x15\a\xa1\x1d\x19\xb5\\8\xad}\xd9\x15F\xa9ھ\xa2<`E\x81\xad\xc5\xf2\xfe\xfdI\x95\xa7\x9c\xfe\xd3`11\"\xcf\xe9\xdbdL\xed;9زK\xf3\xfb\x8eݾ\xdbt\xcd5\xfd4&\xe2\xfb\x0e\xbaL\x00s\x9a\xad\a\xb09\xcc4\xc0\v9\xb8\xaf\x9b\x7f\b\x18I\xdb<\xf2\xd2\xf5\x9c\x1c:\xa1е2\xa90\x9e\xd1\xfe\xeb\xa2l\xbe.\n]ܴOwU\x914%3\xd5\x1d\xeb\xaa9\xdf@\xec\xda\xc79\x8d\xed\xc9\xf5\xfa\n\u0530\x04ܢ\x04\xaa{\x19\x17\x14\xbb=\xc9\f\x80\x1d\xa7\x12\x83Xx+\xe8\x1a\"]\xf3,ۙ:mɌ\x12\xa6h\xf6-\x8d٧\x90_\xd18\x91I\x1a\xbea\n\x19\x8e\f\xa5\xb9ɦ\x90\xc7kGf\x80\x81\x0eD\"n\x1c\x02\xad\xb3\x95\x94\xcd+\x1b4\x86U\xa7\x10\xedsX\x15\xdahq\v\xb05\xa5QC\xd6~0\x11h/\x82+y\x1aS/B\xd2A\a\xfcbN\xbe\xac\xce\xe0\xe5ˊ\x0e\xf9\xb2\xfa\xbd\xbc\xa0tM\xae\xb0bΪ̰\xe0\xd2\xfd\x9a\x19\xdfqY\xaa\xdd\xf4~\x1d\x11\xb5e\xb6>!\xe83\xb3u\x17G7N\b\xbfg\x92_\xc6\xd4l\x8d\x04\x1c\x7fT\x9a\xe9\xfb<\xa7أ5\xb98\x8f\xe7ܙC\x9a\x7f\xc2]f\xb4\x8bK\x99\xa9\xe7\x18\xec2S\x93g\xd0t2\xb4\xd2r\x98\xd2\xcdei\xf6/\x8d\x99\xb9\x7f\xf8(p\x91\x9e#\x7fׄ\xb9\xbe)W+\xd1E6\xffB(]\xb3FMF\xf0o\x90\xa3~\x05\xe5݉\xc52\x84\x93\xfd}\xb2\xef)\xcd\xe1\xa5\xe6\xa6k#v\xe5Z\xc9M+\xd8{/\xcb)l\xedqk\xfc<4u\x92\xe3\xfd\xb7\xfe\xbd6\xdf\xce\xc9=\xba\x0e\xbf\xe9\xf3\xe9h\xbe\x7f\x87\xfd6'\x1c\t\f\xdd\xf5~|8\xb3\x0e}|\xe8\xae\"/QZ*\xad\xf7Or\xfb\x8aƷxs\xba\x1c\xb7\xb6/+\xc2\x06\xaf\xf8W\x15\xa5\x03\n'ҵ\xf8G\x05\xb9\xa4hE`@\x10\xe4\x1f\x81\x96\xe3g\xdf\xdb\xfe\x15\x99\xd9\xf8\x12U\xd4LV\x98\xab\xf4\x94\xa4\x1c\xc0\xe7\x10\x97\xe7_C\x81\xbeg\xea\x95\xf5\xaaɠ\xe7\xbcLh\xc7^b:\xe2\xd6\xfd\xd3\xe0\x02\xfe\xfb\xff\x9b\xdf\x02\x00\x00\xff\xff\xcc\b\u008b\xfc#\x00\x00"), } var CRDs = crds() diff --git a/pkg/apis/velero/v2alpha1/data_download_types.go b/pkg/apis/velero/v2alpha1/data_download_types.go index 3a700661a0..ef85da147b 100644 --- a/pkg/apis/velero/v2alpha1/data_download_types.go +++ b/pkg/apis/velero/v2alpha1/data_download_types.go @@ -54,6 +54,10 @@ type DataDownloadSpec struct { // OperationTimeout specifies the time used to wait internal operations, // before returning error as timeout. OperationTimeout metav1.Duration `json:"operationTimeout"` + + // NodeOS is OS of the node where the DataDonwload is processed. + // +optional + NodeOS NodeOS `json:"nodeOS,omitempty"` } // TargetVolumeSpec is the specification for a target PVC. diff --git a/pkg/apis/velero/v2alpha1/data_upload_types.go b/pkg/apis/velero/v2alpha1/data_upload_types.go index 546caa05e9..140a94ec35 100644 --- a/pkg/apis/velero/v2alpha1/data_upload_types.go +++ b/pkg/apis/velero/v2alpha1/data_upload_types.go @@ -96,6 +96,16 @@ const ( DataUploadPhaseFailed DataUploadPhase = "Failed" ) +// NodeOS represents OS of a node. +// +kubebuilder:validation:Enum=auto;linux;windows +type NodeOS string + +const ( + NodeOSLinux NodeOS = "linux" + NodeOSWindows NodeOS = "windows" + NodeOSAuto NodeOS = "auto" +) + // DataUploadStatus is the current status of a DataUpload. type DataUploadStatus struct { // Phase is the current state of the DataUpload. @@ -144,7 +154,12 @@ type DataUploadStatus struct { // Node is name of the node where the DataUpload is processed. // +optional Node string `json:"node,omitempty"` - // Node is name of the node where the DataUpload is prepared. + + // NodeOS is OS of the node where the DataUpload is processed. + // +optional + NodeOS NodeOS `json:"nodeOS,omitempty"` + + // AcceptedByNode is name of the node where the DataUpload is prepared. // +optional AcceptedByNode string `json:"acceptedByNode,omitempty"` @@ -221,4 +236,8 @@ type DataUploadResult struct { // +optional // +nullable DataMoverResult *map[string]string `json:"dataMoverResult,omitempty"` + + // NodeOS is OS of the node where the DataUpload is processed. + // +optional + NodeOS NodeOS `json:"nodeOS,omitempty"` } diff --git a/pkg/builder/data_download_builder.go b/pkg/builder/data_download_builder.go index 9364022bd5..5b23a9dcc4 100644 --- a/pkg/builder/data_download_builder.go +++ b/pkg/builder/data_download_builder.go @@ -148,6 +148,12 @@ func (d *DataDownloadBuilder) Node(node string) *DataDownloadBuilder { return d } +// NodeOS sets the DataDownload's Node OS. +func (d *DataDownloadBuilder) NodeOS(nodeOS velerov2alpha1api.NodeOS) *DataDownloadBuilder { + d.object.Spec.NodeOS = nodeOS + return d +} + // AcceptedByNode sets the DataDownload's AcceptedByNode. func (d *DataDownloadBuilder) AcceptedByNode(node string) *DataDownloadBuilder { d.object.Status.AcceptedByNode = node diff --git a/pkg/builder/data_upload_builder.go b/pkg/builder/data_upload_builder.go index b4fa72e438..b77566bf66 100644 --- a/pkg/builder/data_upload_builder.go +++ b/pkg/builder/data_upload_builder.go @@ -151,6 +151,12 @@ func (d *DataUploadBuilder) Node(node string) *DataUploadBuilder { return d } +// NodeOS sets the DataUpload's Node OS. +func (d *DataUploadBuilder) NodeOS(nodeOS velerov2alpha1api.NodeOS) *DataUploadBuilder { + d.object.Status.NodeOS = nodeOS + return d +} + // AcceptedByNode sets the DataUpload's AcceptedByNode. func (d *DataUploadBuilder) AcceptedByNode(node string) *DataUploadBuilder { d.object.Status.AcceptedByNode = node diff --git a/pkg/cmd/cli/datamover/restore.go b/pkg/cmd/cli/datamover/restore.go index 244060cc9a..4730cf9035 100644 --- a/pkg/cmd/cli/datamover/restore.go +++ b/pkg/cmd/cli/datamover/restore.go @@ -160,7 +160,24 @@ func newdataMoverRestore(logger logrus.FieldLogger, factory client.Factory, conf return nil, errors.Wrap(err, "error to create client") } - cache, err := ctlcache.New(clientConfig, cacheOption) + var cache ctlcache.Cache + retry := 10 + for { + cache, err = ctlcache.New(clientConfig, cacheOption) + if err == nil { + break + } + + retry-- + if retry == 0 { + break + } + + logger.WithError(err).Warn("Failed to create client cache, need retry") + + time.Sleep(time.Second) + } + if err != nil { cancelFunc() return nil, errors.Wrap(err, "error to create client cache") diff --git a/pkg/controller/data_download_controller.go b/pkg/controller/data_download_controller.go index bbd45b97e8..3d5b2965a7 100644 --- a/pkg/controller/data_download_controller.go +++ b/pkg/controller/data_download_controller.go @@ -183,28 +183,15 @@ func (r *DataDownloadReconciler) Reconcile(ctx context.Context, req ctrl.Request return ctrl.Result{}, nil } - hostingPodLabels := map[string]string{velerov1api.DataDownloadLabel: dd.Name} - for _, k := range util.ThirdPartyLabels { - if v, err := nodeagent.GetLabelValue(ctx, r.kubeClient, dd.Namespace, k, kube.NodeOSLinux); err != nil { - if err != nodeagent.ErrNodeAgentLabelNotFound { - log.WithError(err).Warnf("Failed to check node-agent label, skip adding host pod label %s", k) - } - } else { - hostingPodLabels[k] = v - } + exposeParam, err := r.setupExposeParam(dd) + if err != nil { + return r.errorOut(ctx, dd, err, "failed to set exposer parameters", log) } // Expose() will trigger to create one pod whose volume is restored by a given volume snapshot, // but the pod maybe is not in the same node of the current controller, so we need to return it here. // And then only the controller who is in the same node could do the rest work. - err = r.restoreExposer.Expose(ctx, getDataDownloadOwnerObject(dd), exposer.GenericRestoreExposeParam{ - TargetPVCName: dd.Spec.TargetVolume.PVC, - SourceNamespace: dd.Spec.TargetVolume.Namespace, - HostingPodLabels: hostingPodLabels, - Resources: r.podResources, - ExposeTimeout: dd.Spec.OperationTimeout.Duration, - RestorePVCConfig: r.restorePVCConfig, - }) + err = r.restoreExposer.Expose(ctx, getDataDownloadOwnerObject(dd), exposeParam) if err != nil { if err := r.client.Get(ctx, req.NamespacedName, dd); err != nil { if !apierrors.IsNotFound(err) { @@ -243,7 +230,7 @@ func (r *DataDownloadReconciler) Reconcile(ctx context.Context, req ctrl.Request log.Debugf("Data download is been canceled %s in Phase %s", dd.GetName(), dd.Status.Phase) r.tryCancelAcceptedDataDownload(ctx, dd, "") } else if peekErr := r.restoreExposer.PeekExposed(ctx, getDataDownloadOwnerObject(dd)); peekErr != nil { - r.tryCancelAcceptedDataDownload(ctx, dd, fmt.Sprintf("found a dataupload %s/%s with expose error: %s. mark it as cancel", dd.Namespace, dd.Name, peekErr)) + r.tryCancelAcceptedDataDownload(ctx, dd, fmt.Sprintf("found a datadownload %s/%s with expose error: %s. mark it as cancel", dd.Namespace, dd.Name, peekErr)) log.Errorf("Cancel dd %s/%s because of expose error %s", dd.Namespace, dd.Name, peekErr) } else if dd.Status.AcceptedTimestamp != nil { if time.Since(dd.Status.AcceptedTimestamp.Time) >= r.preparingTimeout { @@ -737,6 +724,42 @@ func (r *DataDownloadReconciler) closeDataPath(ctx context.Context, ddName strin r.dataPathMgr.RemoveAsyncBR(ddName) } +func (r *DataDownloadReconciler) setupExposeParam(dd *velerov2alpha1api.DataDownload) (exposer.GenericRestoreExposeParam, error) { + log := r.logger.WithField("datadownload", dd.Name) + + nodeOS := string(dd.Spec.NodeOS) + if nodeOS == "" { + log.Info("nodeOS is empty in DD, fallback to linux") + nodeOS = kube.NodeOSLinux + } + + if err := kube.HasNodeWithOS(context.Background(), nodeOS, r.kubeClient.CoreV1()); err != nil { + return exposer.GenericRestoreExposeParam{}, errors.Wrapf(err, "no appropriate node to run datadownload %s/%s", dd.Namespace, dd.Name) + } + + hostingPodLabels := map[string]string{velerov1api.DataDownloadLabel: dd.Name} + for _, k := range util.ThirdPartyLabels { + if v, err := nodeagent.GetLabelValue(context.Background(), r.kubeClient, dd.Namespace, k, nodeOS); err != nil { + if err != nodeagent.ErrNodeAgentLabelNotFound { + log.WithError(err).Warnf("Failed to check node-agent label, skip adding host pod label %s", k) + } + } else { + hostingPodLabels[k] = v + } + } + + return exposer.GenericRestoreExposeParam{ + TargetPVCName: dd.Spec.TargetVolume.PVC, + TargetNamespace: dd.Spec.TargetVolume.Namespace, + HostingPodLabels: hostingPodLabels, + Resources: r.podResources, + OperationTimeout: dd.Spec.OperationTimeout.Duration, + ExposeTimeout: r.preparingTimeout, + NodeOS: nodeOS, + RestorePVCConfig: r.restorePVCConfig, + }, nil +} + func getDataDownloadOwnerObject(dd *velerov2alpha1api.DataDownload) v1.ObjectReference { return v1.ObjectReference{ Kind: dd.Kind, diff --git a/pkg/controller/data_download_controller_test.go b/pkg/controller/data_download_controller_test.go index d3d9488958..046103771e 100644 --- a/pkg/controller/data_download_controller_test.go +++ b/pkg/controller/data_download_controller_test.go @@ -53,6 +53,7 @@ import ( "github.com/vmware-tanzu/velero/pkg/nodeagent" velerotest "github.com/vmware-tanzu/velero/pkg/test" "github.com/vmware-tanzu/velero/pkg/uploader" + "github.com/vmware-tanzu/velero/pkg/util/kube" exposermockes "github.com/vmware-tanzu/velero/pkg/exposer/mocks" ) @@ -67,7 +68,7 @@ func dataDownloadBuilder() *builder.DataDownloadBuilder { PV: "test-pv", PVC: "test-pvc", Namespace: "test-ns", - }) + }).NodeOS(velerov2alpha1api.NodeOS("linux")) } func initDataDownloadReconciler(objects []runtime.Object, needError ...bool) (*DataDownloadReconciler, error) { @@ -167,6 +168,8 @@ func TestDataDownloadReconcile(t *testing.T) { }, } + node := builder.ForNode("fake-node").Labels(map[string]string{kube.NodeOSLabel: kube.NodeOSLinux}).Result() + tests := []struct { name string dd *velerov2alpha1api.DataDownload @@ -326,9 +329,15 @@ func TestDataDownloadReconcile(t *testing.T) { }, { name: "Restore is exposed", - dd: dataDownloadBuilder().Result(), + dd: dataDownloadBuilder().NodeOS(velerov2alpha1api.NodeOSLinux).Result(), targetPVC: builder.ForPersistentVolumeClaim("test-ns", "test-pvc").Result(), }, + { + name: "Expected node doesn't exist", + dd: dataDownloadBuilder().NodeOS(velerov2alpha1api.NodeOSWindows).Result(), + targetPVC: builder.ForPersistentVolumeClaim("test-ns", "test-pvc").Result(), + expectedStatusMsg: "no appropriate node to run datadownload", + }, { name: "Get empty restore exposer", dd: dataDownloadBuilder().Phase(velerov2alpha1api.DataDownloadPhasePrepared).Result(), @@ -388,9 +397,9 @@ func TestDataDownloadReconcile(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - var objs []runtime.Object + objs := []runtime.Object{daemonSet, node} if test.targetPVC != nil { - objs = []runtime.Object{test.targetPVC, daemonSet} + objs = append(objs, test.targetPVC) } r, err := initDataDownloadReconciler(objs, test.needErrs...) require.NoError(t, err) diff --git a/pkg/controller/data_upload_controller.go b/pkg/controller/data_upload_controller.go index 66f5b67f76..3185373c14 100644 --- a/pkg/controller/data_upload_controller.go +++ b/pkg/controller/data_upload_controller.go @@ -285,6 +285,10 @@ func (r *DataUploadReconciler) Reconcile(ctx context.Context, req ctrl.Request) return ctrl.Result{}, nil } + if res.ByPod.NodeOS == nil { + return r.errorOut(ctx, du, errors.New("unsupported ambiguous node OS"), "invalid expose result", log) + } + log.Info("Exposed snapshot is ready and creating data path routine") // Need to first create file system BR and get data path instance then update data upload status @@ -317,6 +321,7 @@ func (r *DataUploadReconciler) Reconcile(ctx context.Context, req ctrl.Request) original := du.DeepCopy() du.Status.Phase = velerov2alpha1api.DataUploadPhaseInProgress du.Status.StartTimestamp = &metav1.Time{Time: r.Clock.Now()} + du.Status.NodeOS = velerov2alpha1api.NodeOS(*res.ByPod.NodeOS) if err := r.client.Patch(ctx, du, client.MergeFrom(original)); err != nil { log.WithError(err).Warnf("Failed to update dataupload %s to InProgress, will data path close and retry", du.Name) @@ -792,6 +797,8 @@ func (r *DataUploadReconciler) closeDataPath(ctx context.Context, duName string) } func (r *DataUploadReconciler) setupExposeParam(du *velerov2alpha1api.DataUpload) (interface{}, error) { + log := r.logger.WithField("dataupload", du.Name) + if du.Spec.SnapshotType == velerov2alpha1api.SnapshotTypeCSI { pvc := &corev1.PersistentVolumeClaim{} err := r.client.Get(context.Background(), types.NamespacedName{ @@ -803,7 +810,7 @@ func (r *DataUploadReconciler) setupExposeParam(du *velerov2alpha1api.DataUpload return nil, errors.Wrapf(err, "failed to get PVC %s/%s", du.Spec.SourceNamespace, du.Spec.SourcePVC) } - nodeOS, err := kube.GetPVCAttachingNodeOS(pvc, r.kubeClient.CoreV1(), r.kubeClient.StorageV1(), r.logger) + nodeOS, err := kube.GetPVCAttachingNodeOS(pvc, r.kubeClient.CoreV1(), r.kubeClient.StorageV1(), log) if err != nil { return nil, errors.Wrapf(err, "failed to get attaching node OS for PVC %s/%s", du.Spec.SourceNamespace, du.Spec.SourcePVC) } @@ -821,7 +828,7 @@ func (r *DataUploadReconciler) setupExposeParam(du *velerov2alpha1api.DataUpload for _, k := range util.ThirdPartyLabels { if v, err := nodeagent.GetLabelValue(context.Background(), r.kubeClient, du.Namespace, k, nodeOS); err != nil { if err != nodeagent.ErrNodeAgentLabelNotFound { - r.logger.WithError(err).Warnf("Failed to check node-agent label, skip adding host pod label %s", k) + log.WithError(err).Warnf("Failed to check node-agent label, skip adding host pod label %s", k) } } else { hostingPodLabels[k] = v @@ -843,6 +850,7 @@ func (r *DataUploadReconciler) setupExposeParam(du *velerov2alpha1api.DataUpload NodeOS: nodeOS, }, nil } + return nil, nil } diff --git a/pkg/controller/data_upload_controller_test.go b/pkg/controller/data_upload_controller_test.go index f480a692c5..d5d2fbe138 100644 --- a/pkg/controller/data_upload_controller_test.go +++ b/pkg/controller/data_upload_controller_test.go @@ -166,6 +166,7 @@ func initDataUploaderReconcilerWithError(needError ...error) (*DataUploadReconci RestoreSize: &restoreSize, }, } + daemonSet := &appsv1.DaemonSet{ ObjectMeta: metav1.ObjectMeta{ Namespace: "velero", @@ -265,9 +266,10 @@ func dataUploadBuilder() *builder.DataUploadBuilder { } type fakeSnapshotExposer struct { - kubeClient kbclient.Client - clock clock.WithTickerAndDelayedExecution - peekErr error + kubeClient kbclient.Client + clock clock.WithTickerAndDelayedExecution + ambigousNodeOS bool + peekErr error } func (f *fakeSnapshotExposer) Expose(ctx context.Context, ownerObject corev1.ObjectReference, param interface{}) error { @@ -296,7 +298,13 @@ func (f *fakeSnapshotExposer) GetExposed(ctx context.Context, du corev1.ObjectRe if err != nil { return nil, err } - return &exposer.ExposeResult{ByPod: exposer.ExposeByPod{HostingPod: pod, VolumeName: dataUploadName}}, nil + + nodeOS := "linux" + pNodeOS := &nodeOS + if f.ambigousNodeOS { + pNodeOS = nil + } + return &exposer.ExposeResult{ByPod: exposer.ExposeByPod{HostingPod: pod, VolumeName: dataUploadName, NodeOS: pNodeOS}}, nil } func (f *fakeSnapshotExposer) PeekExposed(ctx context.Context, ownerObject corev1.ObjectReference) error { @@ -350,6 +358,8 @@ func TestReconcile(t *testing.T) { expectedRequeue ctrl.Result expectedErrMsg string needErrs []bool + removeNode bool + ambigousNodeOS bool peekErr error notCreateFSBR bool fsBRInitErr error @@ -359,25 +369,29 @@ func TestReconcile(t *testing.T) { name: "Dataupload is not initialized", du: builder.ForDataUpload("unknown-ns", "unknown-name").Result(), expectedRequeue: ctrl.Result{}, - }, { + }, + { name: "Error get Dataupload", du: builder.ForDataUpload(velerov1api.DefaultNamespace, "unknown-name").Result(), expectedRequeue: ctrl.Result{}, expectedErrMsg: "getting DataUpload: Get error", needErrs: []bool{true, false, false, false}, - }, { + }, + { name: "Unsupported data mover type", du: dataUploadBuilder().DataMover("unknown type").Result(), expected: dataUploadBuilder().Phase("").Result(), expectedRequeue: ctrl.Result{}, - }, { + }, + { name: "Unknown type of snapshot exposer is not initialized", du: dataUploadBuilder().SnapshotType("unknown type").Result(), expectedProcessed: true, expected: dataUploadBuilder().Phase(velerov2alpha1api.DataUploadPhaseFailed).Result(), expectedRequeue: ctrl.Result{}, expectedErrMsg: "unknown type type of snapshot exposer is not exist", - }, { + }, + { name: "Dataupload should be accepted", du: dataUploadBuilder().Result(), pod: builder.ForPod("fake-ns", dataUploadName).Volumes(&corev1.Volume{Name: "test-pvc"}).Result(), @@ -394,6 +408,27 @@ func TestReconcile(t *testing.T) { expectedRequeue: ctrl.Result{}, expectedErrMsg: "failed to get PVC", }, + { + name: "Dataupload should fail to get PVC attaching node", + du: dataUploadBuilder().Result(), + pod: builder.ForPod("fake-ns", dataUploadName).Volumes(&corev1.Volume{Name: "test-pvc"}).Result(), + pvc: builder.ForPersistentVolumeClaim("fake-ns", "test-pvc").StorageClass("fake-sc").Result(), + expectedProcessed: true, + expected: dataUploadBuilder().Phase(velerov2alpha1api.DataUploadPhaseFailed).Result(), + expectedRequeue: ctrl.Result{}, + expectedErrMsg: "error to get storage class", + }, + { + name: "Dataupload should fail because expected node doesn't exist", + du: dataUploadBuilder().Result(), + pod: builder.ForPod("fake-ns", dataUploadName).Volumes(&corev1.Volume{Name: "test-pvc"}).Result(), + pvc: builder.ForPersistentVolumeClaim("fake-ns", "test-pvc").Result(), + removeNode: true, + expectedProcessed: true, + expected: dataUploadBuilder().Phase(velerov2alpha1api.DataUploadPhaseFailed).Result(), + expectedRequeue: ctrl.Result{}, + expectedErrMsg: "no appropriate node to run data upload", + }, { name: "Dataupload should be prepared", du: dataUploadBuilder().SnapshotType(fakeSnapshotType).Result(), @@ -407,6 +442,15 @@ func TestReconcile(t *testing.T) { expected: dataUploadBuilder().Phase(velerov2alpha1api.DataUploadPhaseInProgress).Result(), expectedRequeue: ctrl.Result{}, }, + { + name: "Dataupload should fail if expose returns ambigous nodeOS", + pod: builder.ForPod(velerov1api.DefaultNamespace, dataUploadName).Volumes(&corev1.Volume{Name: "dataupload-1"}).Result(), + du: dataUploadBuilder().Phase(velerov2alpha1api.DataUploadPhasePrepared).SnapshotType(fakeSnapshotType).Result(), + ambigousNodeOS: true, + expectedProcessed: true, + expected: dataUploadBuilder().Phase(velerov2alpha1api.DataUploadPhaseFailed).Result(), + expectedErrMsg: "unsupported ambiguous node OS", + }, { name: "Dataupload with not enabled cancel", pod: builder.ForPod(velerov1api.DefaultNamespace, dataUploadName).Volumes(&corev1.Volume{Name: "dataupload-1"}).Result(), @@ -557,6 +601,11 @@ func TestReconcile(t *testing.T) { require.NoError(t, err) } + if test.removeNode { + err = r.kubeClient.CoreV1().Nodes().Delete(ctx, "fake-node", metav1.DeleteOptions{}) + require.NoError(t, err) + } + if test.dataMgr != nil { r.dataPathMgr = test.dataMgr } else { @@ -564,7 +613,7 @@ func TestReconcile(t *testing.T) { } if test.du.Spec.SnapshotType == fakeSnapshotType { - r.snapshotExposerList = map[velerov2alpha1api.SnapshotType]exposer.SnapshotExposer{fakeSnapshotType: &fakeSnapshotExposer{r.client, r.Clock, test.peekErr}} + r.snapshotExposerList = map[velerov2alpha1api.SnapshotType]exposer.SnapshotExposer{fakeSnapshotType: &fakeSnapshotExposer{r.client, r.Clock, test.ambigousNodeOS, test.peekErr}} } else if test.du.Spec.SnapshotType == velerov2alpha1api.SnapshotTypeCSI { r.snapshotExposerList = map[velerov2alpha1api.SnapshotType]exposer.SnapshotExposer{velerov2alpha1api.SnapshotTypeCSI: exposer.NewCSISnapshotExposer(r.kubeClient, r.csiSnapshotClient, velerotest.NewLogger())} } diff --git a/pkg/exposer/csi_snapshot.go b/pkg/exposer/csi_snapshot.go index 043462792d..b4543ee530 100644 --- a/pkg/exposer/csi_snapshot.go +++ b/pkg/exposer/csi_snapshot.go @@ -281,10 +281,16 @@ func (e *csiSnapshotExposer) GetExposed(ctx context.Context, ownerObject corev1. curLog.WithField("pod", pod.Name).Infof("Backup volume is found in pod at index %v", i) + var nodeOS *string + if os, found := pod.Spec.NodeSelector[kube.NodeOSLabel]; found { + nodeOS = &os + } + return &ExposeResult{ByPod: ExposeByPod{ HostingPod: pod, HostingContainer: containerName, VolumeName: volumeName, + NodeOS: nodeOS, }}, nil } diff --git a/pkg/exposer/generic_restore.go b/pkg/exposer/generic_restore.go index f2c540d6ed..b332c611a3 100644 --- a/pkg/exposer/generic_restore.go +++ b/pkg/exposer/generic_restore.go @@ -40,8 +40,8 @@ type GenericRestoreExposeParam struct { // TargetPVCName is the target volume name to be restored TargetPVCName string - // SourceNamespace is the original namespace of the volume that the snapshot is taken for - SourceNamespace string + // TargetNamespace is the namespace of the volume to be restored + TargetNamespace string // HostingPodLabels is the labels that are going to apply to the hosting pod HostingPodLabels map[string]string @@ -52,6 +52,12 @@ type GenericRestoreExposeParam struct { // ExposeTimeout specifies the timeout for the entire expose process ExposeTimeout time.Duration + // OperationTimeout specifies the time wait for resources operations in Expose + OperationTimeout time.Duration + + // NodeOS specifies the OS of node that the volume should be attached + NodeOS string + // RestorePVCConfig is the config for restorePVC (intermediate PVC) of generic restore RestorePVCConfig nodeagent.RestorePVC } @@ -99,21 +105,21 @@ func (e *genericRestoreExposer) Expose(ctx context.Context, ownerObject corev1.O curLog := e.log.WithFields(logrus.Fields{ "owner": ownerObject.Name, "target PVC": param.TargetPVCName, - "source namespace": param.SourceNamespace, + "target namespace": param.TargetNamespace, }) - selectedNode, targetPVC, err := kube.WaitPVCConsumed(ctx, e.kubeClient.CoreV1(), param.TargetPVCName, param.SourceNamespace, e.kubeClient.StorageV1(), param.ExposeTimeout, param.RestorePVCConfig.IgnoreDelayBinding) + selectedNode, targetPVC, err := kube.WaitPVCConsumed(ctx, e.kubeClient.CoreV1(), param.TargetPVCName, param.TargetNamespace, e.kubeClient.StorageV1(), param.ExposeTimeout, param.RestorePVCConfig.IgnoreDelayBinding) if err != nil { - return errors.Wrapf(err, "error to wait target PVC consumed, %s/%s", param.SourceNamespace, param.TargetPVCName) + return errors.Wrapf(err, "error to wait target PVC consumed, %s/%s", param.TargetNamespace, param.TargetPVCName) } curLog.WithField("target PVC", param.TargetPVCName).WithField("selected node", selectedNode).Info("Target PVC is consumed") if kube.IsPVCBound(targetPVC) { - return errors.Errorf("Target PVC %s/%s has already been bound, abort", param.SourceNamespace, param.TargetPVCName) + return errors.Errorf("Target PVC %s/%s has already been bound, abort", param.TargetNamespace, param.TargetPVCName) } - restorePod, err := e.createRestorePod(ctx, ownerObject, targetPVC, param.ExposeTimeout, param.HostingPodLabels, selectedNode, param.Resources) + restorePod, err := e.createRestorePod(ctx, ownerObject, targetPVC, param.OperationTimeout, param.HostingPodLabels, selectedNode, param.Resources, param.NodeOS) if err != nil { return errors.Wrapf(err, "error to create restore pod") } @@ -274,19 +280,19 @@ func (e *genericRestoreExposer) CleanUp(ctx context.Context, ownerObject corev1. kube.DeletePVAndPVCIfAny(ctx, e.kubeClient.CoreV1(), restorePVCName, ownerObject.Namespace, 0, e.log) } -func (e *genericRestoreExposer) RebindVolume(ctx context.Context, ownerObject corev1.ObjectReference, targetPVCName string, sourceNamespace string, timeout time.Duration) error { +func (e *genericRestoreExposer) RebindVolume(ctx context.Context, ownerObject corev1.ObjectReference, targetPVCName string, targetNamespace string, timeout time.Duration) error { restorePodName := ownerObject.Name restorePVCName := ownerObject.Name curLog := e.log.WithFields(logrus.Fields{ "owner": ownerObject.Name, "target PVC": targetPVCName, - "source namespace": sourceNamespace, + "target namespace": targetNamespace, }) - targetPVC, err := e.kubeClient.CoreV1().PersistentVolumeClaims(sourceNamespace).Get(ctx, targetPVCName, metav1.GetOptions{}) + targetPVC, err := e.kubeClient.CoreV1().PersistentVolumeClaims(targetNamespace).Get(ctx, targetPVCName, metav1.GetOptions{}) if err != nil { - return errors.Wrapf(err, "error to get target PVC %s/%s", sourceNamespace, targetPVCName) + return errors.Wrapf(err, "error to get target PVC %s/%s", targetNamespace, targetPVCName) } restorePV, err := kube.WaitPVCBound(ctx, e.kubeClient.CoreV1(), e.kubeClient.CoreV1(), restorePVCName, ownerObject.Namespace, timeout) @@ -368,7 +374,7 @@ func (e *genericRestoreExposer) RebindVolume(ctx context.Context, ownerObject co } func (e *genericRestoreExposer) createRestorePod(ctx context.Context, ownerObject corev1.ObjectReference, targetPVC *corev1.PersistentVolumeClaim, - operationTimeout time.Duration, label map[string]string, selectedNode string, resources corev1.ResourceRequirements) (*corev1.Pod, error) { + operationTimeout time.Duration, label map[string]string, selectedNode string, resources corev1.ResourceRequirements, nodeType string) (*corev1.Pod, error) { restorePodName := ownerObject.Name restorePVCName := ownerObject.Name @@ -409,7 +415,28 @@ func (e *genericRestoreExposer) createRestorePod(ctx context.Context, ownerObjec args = append(args, podInfo.logFormatArgs...) args = append(args, podInfo.logLevelArgs...) - userID := int64(0) + var securityCtx *corev1.PodSecurityContext + nodeSelector := map[string]string{} + podOS := corev1.PodOS{} + if nodeType == kube.NodeOSWindows { + userID := "ContainerAdministrator" + securityCtx = &corev1.PodSecurityContext{ + WindowsOptions: &corev1.WindowsSecurityContextOptions{ + RunAsUserName: &userID, + }, + } + + nodeSelector[kube.NodeOSLabel] = kube.NodeOSWindows + podOS.Name = kube.NodeOSWindows + } else { + userID := int64(0) + securityCtx = &corev1.PodSecurityContext{ + RunAsUser: &userID, + } + + nodeSelector[kube.NodeOSLabel] = kube.NodeOSLinux + podOS.Name = kube.NodeOSLinux + } pod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ @@ -427,6 +454,8 @@ func (e *genericRestoreExposer) createRestorePod(ctx context.Context, ownerObjec Labels: label, }, Spec: corev1.PodSpec{ + NodeSelector: nodeSelector, + OS: &podOS, Containers: []corev1.Container{ { Name: containerName, @@ -450,9 +479,7 @@ func (e *genericRestoreExposer) createRestorePod(ctx context.Context, ownerObjec Volumes: volumes, NodeName: selectedNode, RestartPolicy: corev1.RestartPolicyNever, - SecurityContext: &corev1.PodSecurityContext{ - RunAsUser: &userID, - }, + SecurityContext: securityCtx, }, } diff --git a/pkg/exposer/generic_restore_test.go b/pkg/exposer/generic_restore_test.go index 338d58b52b..15f8c1615b 100644 --- a/pkg/exposer/generic_restore_test.go +++ b/pkg/exposer/generic_restore_test.go @@ -31,7 +31,6 @@ import ( velerotest "github.com/vmware-tanzu/velero/pkg/test" appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" corev1api "k8s.io/api/core/v1" clientTesting "k8s.io/client-go/testing" ) @@ -76,9 +75,9 @@ func TestRestoreExpose(t *testing.T) { APIVersion: appsv1.SchemeGroupVersion.String(), }, Spec: appsv1.DaemonSetSpec{ - Template: corev1.PodTemplateSpec{ - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ + Template: corev1api.PodTemplateSpec{ + Spec: corev1api.PodSpec{ + Containers: []corev1api.Container{ { Image: "fake-image", }, @@ -93,21 +92,21 @@ func TestRestoreExpose(t *testing.T) { kubeClientObj []runtime.Object ownerRestore *velerov1.Restore targetPVCName string - sourceNamespace string + targetNamespace string kubeReactors []reactor err string }{ { name: "wait target pvc consumed fail", targetPVCName: "fake-target-pvc", - sourceNamespace: "fake-ns", + targetNamespace: "fake-ns", ownerRestore: restore, err: "error to wait target PVC consumed, fake-ns/fake-target-pvc: error to wait for PVC: error to get pvc fake-ns/fake-target-pvc: persistentvolumeclaims \"fake-target-pvc\" not found", }, { name: "target pvc is already bound", targetPVCName: "fake-target-pvc", - sourceNamespace: "fake-ns", + targetNamespace: "fake-ns", ownerRestore: restore, kubeClientObj: []runtime.Object{ targetPVCObjBound, @@ -117,7 +116,7 @@ func TestRestoreExpose(t *testing.T) { { name: "create restore pod fail", targetPVCName: "fake-target-pvc", - sourceNamespace: "fake-ns", + targetNamespace: "fake-ns", ownerRestore: restore, kubeClientObj: []runtime.Object{ targetPVCObj, @@ -137,7 +136,7 @@ func TestRestoreExpose(t *testing.T) { { name: "create restore pvc fail", targetPVCName: "fake-target-pvc", - sourceNamespace: "fake-ns", + targetNamespace: "fake-ns", ownerRestore: restore, kubeClientObj: []runtime.Object{ targetPVCObj, @@ -182,9 +181,9 @@ func TestRestoreExpose(t *testing.T) { err := exposer.Expose(context.Background(), ownerObject, GenericRestoreExposeParam{ TargetPVCName: test.targetPVCName, - SourceNamespace: test.sourceNamespace, + TargetNamespace: test.targetNamespace, HostingPodLabels: map[string]string{}, - Resources: corev1.ResourceRequirements{}, + Resources: corev1api.ResourceRequirements{}, ExposeTimeout: time.Millisecond}) assert.EqualError(t, err, test.err) }) @@ -244,21 +243,21 @@ func TestRebindVolume(t *testing.T) { kubeClientObj []runtime.Object ownerRestore *velerov1.Restore targetPVCName string - sourceNamespace string + targetNamespace string kubeReactors []reactor err string }{ { name: "get target pvc fail", targetPVCName: "fake-target-pvc", - sourceNamespace: "fake-ns", + targetNamespace: "fake-ns", ownerRestore: restore, err: "error to get target PVC fake-ns/fake-target-pvc: persistentvolumeclaims \"fake-target-pvc\" not found", }, { name: "wait restore pvc bound fail", targetPVCName: "fake-target-pvc", - sourceNamespace: "fake-ns", + targetNamespace: "fake-ns", ownerRestore: restore, kubeClientObj: []runtime.Object{ targetPVCObj, @@ -268,7 +267,7 @@ func TestRebindVolume(t *testing.T) { { name: "retain target pv fail", targetPVCName: "fake-target-pvc", - sourceNamespace: "fake-ns", + targetNamespace: "fake-ns", ownerRestore: restore, kubeClientObj: []runtime.Object{ targetPVCObj, @@ -289,7 +288,7 @@ func TestRebindVolume(t *testing.T) { { name: "delete restore pod fail", targetPVCName: "fake-target-pvc", - sourceNamespace: "fake-ns", + targetNamespace: "fake-ns", ownerRestore: restore, kubeClientObj: []runtime.Object{ targetPVCObj, @@ -311,7 +310,7 @@ func TestRebindVolume(t *testing.T) { { name: "delete restore pvc fail", targetPVCName: "fake-target-pvc", - sourceNamespace: "fake-ns", + targetNamespace: "fake-ns", ownerRestore: restore, kubeClientObj: []runtime.Object{ targetPVCObj, @@ -333,7 +332,7 @@ func TestRebindVolume(t *testing.T) { { name: "rebind target pvc fail", targetPVCName: "fake-target-pvc", - sourceNamespace: "fake-ns", + targetNamespace: "fake-ns", ownerRestore: restore, kubeClientObj: []runtime.Object{ targetPVCObj, @@ -355,7 +354,7 @@ func TestRebindVolume(t *testing.T) { { name: "reset pv binding fail", targetPVCName: "fake-target-pvc", - sourceNamespace: "fake-ns", + targetNamespace: "fake-ns", ownerRestore: restore, kubeClientObj: []runtime.Object{ targetPVCObj, @@ -382,7 +381,7 @@ func TestRebindVolume(t *testing.T) { { name: "wait restore PV bound fail", targetPVCName: "fake-target-pvc", - sourceNamespace: "fake-ns", + targetNamespace: "fake-ns", ownerRestore: restore, kubeClientObj: []runtime.Object{ targetPVCObj, @@ -420,7 +419,7 @@ func TestRebindVolume(t *testing.T) { hookCount = 0 - err := exposer.RebindVolume(context.Background(), ownerObject, test.targetPVCName, test.sourceNamespace, time.Millisecond) + err := exposer.RebindVolume(context.Background(), ownerObject, test.targetPVCName, test.targetNamespace, time.Millisecond) assert.EqualError(t, err, test.err) }) } @@ -526,7 +525,7 @@ func Test_ReastoreDiagnoseExpose(t *testing.T) { }, } - restorePodWithoutNodeName := corev1.Pod{ + restorePodWithoutNodeName := corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1.DefaultNamespace, Name: "fake-restore", @@ -539,19 +538,19 @@ func Test_ReastoreDiagnoseExpose(t *testing.T) { }, }, }, - Status: corev1.PodStatus{ - Phase: corev1.PodPending, - Conditions: []corev1.PodCondition{ + Status: corev1api.PodStatus{ + Phase: corev1api.PodPending, + Conditions: []corev1api.PodCondition{ { - Type: corev1.PodInitialized, - Status: corev1.ConditionTrue, + Type: corev1api.PodInitialized, + Status: corev1api.ConditionTrue, Message: "fake-pod-message", }, }, }, } - restorePodWithNodeName := corev1.Pod{ + restorePodWithNodeName := corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1.DefaultNamespace, Name: "fake-restore", @@ -564,22 +563,22 @@ func Test_ReastoreDiagnoseExpose(t *testing.T) { }, }, }, - Spec: corev1.PodSpec{ + Spec: corev1api.PodSpec{ NodeName: "fake-node", }, - Status: corev1.PodStatus{ - Phase: corev1.PodPending, - Conditions: []corev1.PodCondition{ + Status: corev1api.PodStatus{ + Phase: corev1api.PodPending, + Conditions: []corev1api.PodCondition{ { - Type: corev1.PodInitialized, - Status: corev1.ConditionTrue, + Type: corev1api.PodInitialized, + Status: corev1api.ConditionTrue, Message: "fake-pod-message", }, }, }, } - restorePVCWithoutVolumeName := corev1.PersistentVolumeClaim{ + restorePVCWithoutVolumeName := corev1api.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1.DefaultNamespace, Name: "fake-restore", @@ -592,12 +591,12 @@ func Test_ReastoreDiagnoseExpose(t *testing.T) { }, }, }, - Status: corev1.PersistentVolumeClaimStatus{ - Phase: corev1.ClaimPending, + Status: corev1api.PersistentVolumeClaimStatus{ + Phase: corev1api.ClaimPending, }, } - restorePVCWithVolumeName := corev1.PersistentVolumeClaim{ + restorePVCWithVolumeName := corev1api.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1.DefaultNamespace, Name: "fake-restore", @@ -610,35 +609,35 @@ func Test_ReastoreDiagnoseExpose(t *testing.T) { }, }, }, - Spec: corev1.PersistentVolumeClaimSpec{ + Spec: corev1api.PersistentVolumeClaimSpec{ VolumeName: "fake-pv", }, - Status: corev1.PersistentVolumeClaimStatus{ - Phase: corev1.ClaimPending, + Status: corev1api.PersistentVolumeClaimStatus{ + Phase: corev1api.ClaimPending, }, } - restorePV := corev1.PersistentVolume{ + restorePV := corev1api.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ Name: "fake-pv", }, - Status: corev1.PersistentVolumeStatus{ - Phase: corev1.VolumePending, + Status: corev1api.PersistentVolumeStatus{ + Phase: corev1api.VolumePending, Message: "fake-pv-message", }, } - nodeAgentPod := corev1.Pod{ + nodeAgentPod := corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1.DefaultNamespace, Name: "node-agent-pod-1", Labels: map[string]string{"role": "node-agent"}, }, - Spec: corev1.PodSpec{ + Spec: corev1api.PodSpec{ NodeName: "fake-node", }, - Status: corev1.PodStatus{ - Phase: corev1.PodRunning, + Status: corev1api.PodStatus{ + Phase: corev1api.PodRunning, }, } diff --git a/pkg/exposer/types.go b/pkg/exposer/types.go index d4d8c87300..670c8b661a 100644 --- a/pkg/exposer/types.go +++ b/pkg/exposer/types.go @@ -38,4 +38,5 @@ type ExposeByPod struct { HostingPod *corev1.Pod HostingContainer string VolumeName string + NodeOS *string } diff --git a/pkg/restore/actions/csi/pvc_action.go b/pkg/restore/actions/csi/pvc_action.go index 19c687e8bf..0462bb74c9 100644 --- a/pkg/restore/actions/csi/pvc_action.go +++ b/pkg/restore/actions/csi/pvc_action.go @@ -478,6 +478,7 @@ func newDataDownload( SnapshotID: dataUploadResult.SnapshotID, SourceNamespace: dataUploadResult.SourceNamespace, OperationTimeout: backup.Spec.CSISnapshotTimeout, + NodeOS: dataUploadResult.NodeOS, }, } if restore.Spec.UploaderConfig != nil { diff --git a/pkg/restore/actions/dataupload_retrieve_action.go b/pkg/restore/actions/dataupload_retrieve_action.go index 653f5e3403..d5b922d6f0 100644 --- a/pkg/restore/actions/dataupload_retrieve_action.go +++ b/pkg/restore/actions/dataupload_retrieve_action.go @@ -78,6 +78,7 @@ func (d *DataUploadRetrieveAction) Execute(input *velero.RestoreItemActionExecut SnapshotID: dataUpload.Status.SnapshotID, SourceNamespace: dataUpload.Spec.SourceNamespace, DataMoverResult: dataUpload.Status.DataMoverResult, + NodeOS: dataUpload.Status.NodeOS, } jsonBytes, err := json.Marshal(dataUploadResult)