From 55670bcc2246ef71f385d2eab750454979d5469c Mon Sep 17 00:00:00 2001 From: Roi Kramer Date: Tue, 19 Sep 2023 21:12:36 +0300 Subject: [PATCH 1/8] wip --- .../ocean-kubernetes-controller/.helmignore | 23 ++ charts/ocean-kubernetes-controller/Chart.lock | 6 + charts/ocean-kubernetes-controller/Chart.yaml | 19 ++ .../README.md.gotmpl | 35 +++ .../charts/metrics-server-3.11.0.tgz | Bin 0 -> 8462 bytes .../temp/NOTES.txt | 22 ++ .../ocean-kubernetes-controller/temp/hpa.yaml | 32 +++ .../temp/ingress.yaml | 61 ++++++ .../temp/service.yaml | 15 ++ .../temp/serviceaccount.yaml | 12 + .../templates/_helpers.tpl | 119 ++++++++++ .../templates/deployment.yaml | 167 ++++++++++++++ .../templates/tests/test-connection.yaml | 15 ++ .../ocean-kubernetes-controller/values.yaml | 207 ++++++++++++++++++ 14 files changed, 733 insertions(+) create mode 100644 charts/ocean-kubernetes-controller/.helmignore create mode 100644 charts/ocean-kubernetes-controller/Chart.lock create mode 100644 charts/ocean-kubernetes-controller/Chart.yaml create mode 100644 charts/ocean-kubernetes-controller/README.md.gotmpl create mode 100644 charts/ocean-kubernetes-controller/charts/metrics-server-3.11.0.tgz create mode 100644 charts/ocean-kubernetes-controller/temp/NOTES.txt create mode 100644 charts/ocean-kubernetes-controller/temp/hpa.yaml create mode 100644 charts/ocean-kubernetes-controller/temp/ingress.yaml create mode 100644 charts/ocean-kubernetes-controller/temp/service.yaml create mode 100644 charts/ocean-kubernetes-controller/temp/serviceaccount.yaml create mode 100644 charts/ocean-kubernetes-controller/templates/_helpers.tpl create mode 100644 charts/ocean-kubernetes-controller/templates/deployment.yaml create mode 100644 charts/ocean-kubernetes-controller/templates/tests/test-connection.yaml create mode 100644 charts/ocean-kubernetes-controller/values.yaml diff --git a/charts/ocean-kubernetes-controller/.helmignore b/charts/ocean-kubernetes-controller/.helmignore new file mode 100644 index 0000000..0e8a0eb --- /dev/null +++ b/charts/ocean-kubernetes-controller/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/ocean-kubernetes-controller/Chart.lock b/charts/ocean-kubernetes-controller/Chart.lock new file mode 100644 index 0000000..246508a --- /dev/null +++ b/charts/ocean-kubernetes-controller/Chart.lock @@ -0,0 +1,6 @@ +dependencies: +- name: metrics-server + repository: https://kubernetes-sigs.github.io/metrics-server + version: 3.11.0 +digest: sha256:472915f28d2f1016d5e51ec6c5173c3c01856132a1012dc951242452533b99a9 +generated: "2023-09-19T19:33:55.787057+03:00" diff --git a/charts/ocean-kubernetes-controller/Chart.yaml b/charts/ocean-kubernetes-controller/Chart.yaml new file mode 100644 index 0000000..79b5ecf --- /dev/null +++ b/charts/ocean-kubernetes-controller/Chart.yaml @@ -0,0 +1,19 @@ +apiVersion: v2 +name: ocean-kubernetes-controller +description: A Helm chart for Ocean Controller +type: application +version: 0.1.0 +appVersion: "2.0.16" +maintainers: +- name: spotinst + email: ng-spot-info@netapp.com +icon: https://docs.spot.io/_media/images/spot_mark.png +keywords: +- spot +- ocean +- controller +dependencies: +- name: metrics-server + version: 3.11.0 + repository: https://kubernetes-sigs.github.io/metrics-server + condition: metrics-server.deployChart \ No newline at end of file diff --git a/charts/ocean-kubernetes-controller/README.md.gotmpl b/charts/ocean-kubernetes-controller/README.md.gotmpl new file mode 100644 index 0000000..cf7d1ae --- /dev/null +++ b/charts/ocean-kubernetes-controller/README.md.gotmpl @@ -0,0 +1,35 @@ +{{ template "chart.header" . }} + +{{ template "chart.deprecationWarning" . }} + +{{ template "chart.versionBadge" . }}{{ template "chart.typeBadge" . }}{{ template "chart.appVersionBadge" . }} + +{{ template "chart.description" . }}. + +## Installation + +1. Add the Spot Helm chart repository: + +```sh +helm repo add spot https://charts.spot.io +``` + +2. Update your local Helm chart repository cache: + +```sh +helm repo update +``` + +3. Install `{{ template "chart.name" . }}`: + +```sh +helm install my-release spot/{{ template "chart.name" . }} +``` + +> NOTE: Please configure all required chart values using the `set` command line argument or a `values.yaml` file. + +{{ template "chart.requirementsSection" . }} + +{{ template "chart.valuesSection" . }} + +{{ template "helm-docs.versionFooter" . }} diff --git a/charts/ocean-kubernetes-controller/charts/metrics-server-3.11.0.tgz b/charts/ocean-kubernetes-controller/charts/metrics-server-3.11.0.tgz new file mode 100644 index 0000000000000000000000000000000000000000..8860457ec1efe286aefd1237d6a403256c933800 GIT binary patch literal 8462 zcmV+pA@SZHiwG0|00000|0w_~VMtOiV@ORlOnEsqVl!4SWK%V1T2nbTPgYhoO;>Dc zVQyr3R8em|NM&qo0PMYMbK5x5D7>HfD|+pB-kr$_b@46xs7{_!9w*PPYvYV6&dk=- z)>Z_OEeRWv-~gZ;O+4qfpTdQAQj%>+No?UmVv)^8qtR$I8t6udl2cE(0{I+y_hLr4 z>?}wa{AqKg*X#ApPEPdyyUf*|EFI6q<`A~6ZAGnLu*Nd zB>Ydk2iH|?+&5B46fsF8Wsw*-0EA0AA)cHi;|^tAZ$_diiUHg@001t0q>(=W;``|9 zCjdSQkJDIT7{I}~dILUAND@f6GQtJdTDX9tUUst(CtpC7Cn!J-H#a#SJ0{GmNk__Jw_H&lJ4fJN`f7vi|Ic5ZI3ICw!?-U7jthBRjtiBFvIkB`sBO3Y zC?bk-A#si~8sA_LH~@R8`VB@tM{mg&7QjGP+G09OQ-61k z%SDU>AaNW}Ps^w?Po?U()9-hB&X-nZ`*P<}ng6W{*{lJuV*Y<|diJ6^|DPPa*w6pF zD2MRM&P}H4wfZg#MaMaW_cJQY%<${^)f;!hcu1t0jwUof1@H>JfN+F4;Z#i=LQP;} zgqR3{KF~-qSg?eH#4rwsRI1<6v#T;YIL1s+$@pRb9H&%BzUX}Xt57ybMM5h|3E&WZ znj`1bS9Q)XowCUkZe~p6b1xMz|NI#`#>73(=0y184+j+d01njb9w$LCWC8US1Gt>L zWpc<-U?eTM!z2i<(c>t^0RHuF=MYlOoDy!{z9k_#S_`M-Y5bEU2x<=<)fnpGx#zJY zk^|kU>li&cp(x;HhH|Fnda!Un3?PUZO9Ef<+2g3p7zgZM1CTsHjp#f#@l>Y}-&g-B zR#ng``>$XQ!skEq_wQ$@Tc~FKhOZQ|L|{k5UQQrl5(G*hk#A{HtXI@L2VxLW0aJ{S zYr`uM$|6vRQ{H4U;(%qX?ifQ!K!sG42-61;DjJl;!FT~uiD`vCL)bN9Hs$5>oRTk} zd7{g4f*d0cy8#(vAlw2YTs7xQ(LKay!nlWSR;6o+a3Cc|i-g0+#B1xscR!wAWM-3C zrw3yZzm8OO{s1N<5a=ia@7lDoS`9UQH1Cg5l77dI7CV;Ue0Z6$r&hCOlt%X}OF%AS z>X9H=zztEItT>B?j6bVxUa$mwDm3SdXf-l5tQ5eE%$4=5k%lBv1CvT%i4@eg@h&zLN~ZM;gQX zH`g#nPA91$1VAH!Uc&MEBaPp`x&B4HT@(mWqC0$j1sHiuEs-wHVVp#MfV$lc2tD$n zes2_n$HI_BZ?tS`0%PWvf1Ouapq^bTlBirK!kW}%yCVa>jjCB z$X$<9sl-@u(Aw@p#-*a@=(OiJGbZF)luDu+MpAjMDU29pGe|wHK>-7V%*XO5y3t># z^)BUD8bk`e8X}26Ab~_4DNV`d$OE!~$Y4}LujC_UzEBF)$h4JQRxS2U6lQMZ{6B2E+jRj+5x6 z^tDuaeY()nc9{1t3)C1k%WeIf7oQCDBY=>6x=#2M2hhu({2Y-vp?X28D~g%_iVB|S z1?-QBKSeudF46Fkh>xPnlgKTsV7lp=8{OC+ck2xV2x)YlRqoWtaU5%3ob#y`R2N*= zLoQvP@|Rs1#`c*aDM0DQdaB{*yTtc73gIgLh?kd!iw7;-k&EmBPTo8cfpOyq0; zT?O>Jd7y`jW~}}#ycw#2_V@Sihu2#1QAtVg3In1n9~SvS^&LQroN5(h9`)6;386v4 z@%@aWn6bbgz_F4c;?tEBDX7H*NIJR`O(LINjrd*|g>pG`()v$MjvFVAAXm#uB;nlG zJ|KbijT)_bUaV`E8%ScMQ%0(vJsCi%#%MHQBZICi+9^OvFB~Z4d(Z@GEvc(gtp|>y zeNyj?OKQ+G0x~Ooxt(5^sJj}cYyDhC%`5xb_|Wupl6fKwTH_B))w>rb132l0_GgHp zuJP#fit0xJ4e7ej+R0`>HK2vVPm+@h^`LJaq@VR{9U9;&#bqCcG_rzlO@Bu{uUUx< zf}xhJ0UYb~pJ{Y1Q>Tt&`@4b1skmLSNHMpu7l`juC1xaeQxq_&)W)0ywjILbB*ynN zM3(4TPGp|Bcc|r$Mph9hwwIqcWixqg&(NXmcBj;>>;Q$kRcpK*(gGl*A%^9kYN<7uT^; zG8Zfol53Mc0=zQMW~jC`JXjYOerWo36Q@{S2JIF*`Jll>N;DUci;Z>{`GZ7 zev%KbV}<{xcXm?s|D2wk9q;`=yC}~#Q=DhjZG?N3XU?YR0nmOP$XwL9>!;-RIgjHU z2B6(RbeX!92Aa~(pMgeRkoX7(6;oT!mj}>M-)*WnTkHSU@&93lL5y5YjQ_LK)6@O<-$nWSx%vB@rU#IvkN{>XjGSX^~nLa zIgoa@+F_(-Jm~ylwiSwwPq+oilgNX!WBn%$uan7yeu4w{04y$yeD$ZJiM%krM3U7o zNrGSjza=D43w9q!9BcAB&Og!M*6>o*pqe6Jj2=;oTfsss?osMSPbdazmCex@BWNdi zD&TwVT10sD_FBQJRk?`K`=R4pPEz-iL1MiOsZYyzJ0;xEAb>Hdm9-B<0MW!-YFF_$ zb&KXarjv}NRey5Iy4LsWusskNp8robVeoSOe$8pjm|>7~-dRVT{x+qv^ze?v#hgZR z0tep-_d9V=!DT94U&@;QEXiJ>Q`LB}^M>kB-LY0m43svlS9h)k&U?1nq9U`)v)8R! zj0MC)IQU-&aPZ;a?gL8u6z!^XC7}k)P&&gP?1))c6F*os7ux1^3<0fR!kX)EY$ct7 z6gAGrB#t}f-I{Oac*O)%+h}e|BZU-H1mW;LXybCGDnJoxM{3ZCzIw~qcs{>+!=e@rUuJk z9lP&TnUcP5Yksa_J3q9cjEe0o=%5t&f&JrPEd(vy|32TpT#r50)~u_KfmR=u5)IYK zoj34(#**}lc|qOeF>Yo%Fx&%s+X^UEnCUkm5 z;!J!W&;~C>)~e#y6Uqba#&Qk1W_A9K1-J>4ISVqbXT;k5p6`UK30ary20GzyC}CK4 z;OlLTR9&wkIQGY#{E(ZWR;|gjPEb?x_nnKv22$sIcxhop=kG+oL993UyYv3R5AeOf zaE|{@X>V%%l0if^-fw7DaxZl*H6} z{(p**#%-0DTKElCRG04En~(FI{Li0Ze0GA7XH2aUUVtfQNt_+o z%z2}q5A9a>6^&?^gz45p%drDxsMX181U7XS_Pf%)T4F91bUavnrhbZ;G+BxIaSCVl z&I;@ezz%MmHzL& zI4bM^qvKw0um5*aD*B&~iPxt2^*fSCeL$;O`PEG0RAo8P7wF#rPzpkKS=8Nn{ z#C$Zr=E%nds42?q?|_Q@#|@F*Y_D=3U0F;2>owo~mjSExe@{;;@qb6X(--^m|2rv# zHFMJ((C9x&PcZIgg1D>0VFMFGLElEY*;<+Y{O(ObQ>JHN+O+65qAJ%J{||KltX%(} zo>kZXC#U=IzmxK%?Y}QJuq%d?Yhp}lrjxam+D3b7F|Cbni75s)l1Q(Y^GC+#81nQDXSoXjbAUiIH}h76=chw^-IJ8(W|Oscl3Gs(k@DctINCH1xo~gG<~R z@D+X}`mp4VoTrbVtfT+OH1cUQb=BB+eGJqhh26miu*&{>QD6VR=%4NN|1Qeb^#6}W zXqHa{--&^p3Cgmy<5__P_zovJ63XhqC9QP;>XTjCw#L>tA{qY&J!xki^YwLUcX&O& zYtv@L2aT|Lvr#82{Dnq0KJ?*U}STb~U(h_3-&~ z7UH~OZ^g9t>r^0n!|D-~JLvy=y8%}2|J3w`Ii>M zefdJAUH@lpr-yF;thWFA$JPCx9$+wS1fAYC9oowbyz_i5@%& zq~yL;(tT0Py*PkdSK}={~wn+wUJ8 zAJ@+Rp6&Plc2demdt;H?@~`sL26xi#wZox1(&r6FavM`JZ0Z1Tp8BYuG1G@F1eG%+ z*`v2UX~7}tf;!&Ao7rwD2|p`p9*Q7ku3w|OfruJ19*Qi zEWN3uV=Q!g=}n!SlCYO@*_B~yY3|Bs?kngu63W@@%E0x^cAFt$Ud)13=|s~NChuUe z=%lqnlS{Asw$`Ltrmd=|Y?y&*F5C+5DhG@nTJ<`fjskeI=C%8Ac}1%gUsFtTWtd?! zZ|=scUq1|gdiCM$`PJ)6cj|22|HD~0sJsA^$i%w08{ZizNDaD<%`dsM_UqyK#g<9X zYk>m1$HI_Aew8uZnE4;7y+u%!0r*$tzc&$uki);Xxs3NF?Jg@l(tqj3gyN4XT{wa7Be>;HF9wfi^3t zcv-N2U@b*fjC*$t_!7m|*z5I3%~H7g4%JWT(cI=NtZAIGcE&K4i&vD}e2;K{O}3ns znIE>?;DrX^+wd(Hb>e@Gs+T~~d?jP*6 zF%_i}hZm*a)eQ*!P435aL8rIqTfH5UPXjnRJv}~i z_V)PWF0J~16Nw*eS*ibzde!qE{j>e~znzpX6aUj5AikEH$)u>twzmENXT+RP~c6!?1>;GMp#<4uc}>V^3#F1qULhppF0st+xvu_fVCl*7`E+U0%S`)=Z%?Dq`5fwD^f+v~a> zG!3lK|GgJSwfO)3(LVp*PRi$X?on$^08lYk8|RDGyeLcT)-OJFqzu@gx?fe5$iw?Y z9?#c;{s@|HlDo4BziRx4=1Tf)j{l>hdj9wQ{@;$u=X(Bk+w)hp=TkRtTjW|*IarCI z(X`_+j!a*G*{!M!i~UxV{z@IP#ksq`C__u4V)RNt5qBhE5jKZ5?)xed3Fu7IoqvXZQtkP?*D$%vex{F2< z`J4oWXlnapP>F+FeKx*=T6mnq_@0Ky5($Mc=i>7R*J7CV4z|^Kdi6 z2*wEwd>Tz5Cf-LfMbU8%;r)yXeO!@oDL~9H2w)npv6(+;G<~jzzNB;1+P(0cM80zf z5l*%B4By8bC-f8gM!Ed?hYtJ{1q)!2{!)(lKgdL~z>iWaAWE`DCu=ke#a&Zkl z<-+MqsqE^14f#%I{5$XJf71uEX;=M|{uJ}5n2#=%q-(%A`;XI^3y=BO{TB3AW&FQ; zeg5j|btm*US;zA6e|B_oRQ3NJpZ53hKRYRh@Q>`Wcepabplx*Ij`OebgMWXoZL&eP zJEd}#jJ08(-xco)I?XOhNd@W$paKX8kLWOx=YW$5^{ATn&A~QclROzjcHAuuyVv)K z7bsLpNxC!wl1QeIY6V4OWojr5<^1nIyDq%vbUM}8wB-w~``?apcnFt~kR;IG$QgpZ z?c8yG#EDXR3zmRKBA^CHLwE{8zqz4701=|E)FknJ{d!bx>Y@2Sp(|s8^veo&JC^a3Rr##&I?nk-B3JbOWsYe>(9o)DYHeeU zuq2*x;w#pruoVg`^;e?cH8qEaP|k>`uZf?qAYeDT4|HHrK!p^Bk`!1yjtM6rO5{RG zhhIyMp=-h(4 zMuGLf9&_0^3pDdbCVi3`_Cl@G7Nf>)zf@3=Lv=PK0N+Zvg^|6_e32e!S*qHa*jPrG zGhdSF67ZDyl1!hQ`Gw7qbLyj3cd0g$WC}iVI!7NSoP~vniNtYwHTJjJZ?`ZyI9SQi zyCBS&pC=AmCix1=hP8d0EWCfYh0*2YEt5lz0wekGZpfKmNJ_Yp`F6A8iE;lHM*sTv z#w9-@A^v2QNE`B7_^GI7^f#l_#xlm>2Fdy=i{cawHpqR={B(C>SqD^^pMLX%I$%4@ zZ``;ZRcpSSz)120N6*1-Z4-#_CQW_{>1l_}5%-mqTigE4A#7lNVUK)NsJWPNH(f+C zE%Z->{kJgs{Kv+1eueo)o78H)F0)v1evFU^^gC))L`;G}vzTrhOH^cohv+>>^<;_> zaw5@(M5KNdrPK^SdYs@Z=+T|19noa-+`{{radRjopC0qjY3sD7W4r*V-Rjj*aE;jD z2vRE)pLwFoae^Ep54&dL+AWMbSL)1L-9wBfjC<&2)wwAFyDZzWb_&4?t+<* zKb~LYk+)xgd~UC7N9R{Eg*0Mso^WI4XNjoo(kttUN7oU`E+m229QQTzNldLmZgUf2`GO_jQ=yeu;bT)7 zWswG$kvXc_CM1zs@=*yakwO{3+Tx|c`Z6Yr19qmM!)#b2NuUL#=;xOVZW#Y)HVpNr z*1_KFLu7=D=OMZ^s@!%ry=YZw5?98a@dh!*sA1kkGWoyhoZ<}cI zn%gJbC(;93V}9l0o_ZCxkb>x=9QetYb5Dr(Tl4L#;QVste<*KVe#j5nJgz>ug%8(= z>A8;u34;$0M5raddOcL*B5P=&z*At>?ZqHzUzI$vqQq2ZU~uh~@vg_IR13#@4hL)P z6m9`(L3FD_3Z9&_7t1Y-`n{vm-e%nk4fmNf*!czbf{>)b*IP77usW;uBeeiqVcE^sO;q= zQmYI!9)n5<3!JNu0Y8y24)D1lL;DmoStU~biDZsx^}r%{{g3O;mV7(u6_U%a-oo1? z93xj2eRk!fWWbYe^dQLHi<10J&bNm7z zfl*#_-K#ljxKm*Cs9nlCVt(-=0_#{qX`ILvmQQ`f@{mR}OhU+S4NxljQFXv}wh>p~ zzc5iy^s z8(w6QWn9l{KF@-ypD6#gvw>c^Beivr^~OR$tiT=xJMUJuBl+daVcSg2cPZ3U<^Fb< zUre#pzL}C9no>|tkMljPlmQJj-A6;1Qtmb3y zk*%K9w3%kwbbjMD;uGP!E?YV}(vZBqT1m;hz>`R`UVrFv^Cb`+&rE&vh(1-N%Ud7 zfRO2PJw(CT`oE%%yM@u|=8uk4@*=Meh|{-FLlPBk2SOi+=b_ou8l{K<;8 zIXKmPiV~P{w3Ty%e;X`aziic|+?p2JoZBf$xq&Iwo#u>{9OcO8(+r?~1}7>&?rW`Pz~53k#AEuUHZ_ zX1FU^J97QUlIgeW4?G|@rYZ>F39}s%9(cNzhyqA7QT(r0U`vQ zMo+}6yd67VvHS=O*i;^F+fK3&X>NHWoM<5>)xYXkGh4!_M50<)%{<+lYZs?pVQ;B{rx{X wDTJFF`dn~bwXhFRx@lrHR|ewgjD&sJmwnlnr&a!600030|J=8qQUDSG07YH*+yDRo literal 0 HcmV?d00001 diff --git a/charts/ocean-kubernetes-controller/temp/NOTES.txt b/charts/ocean-kubernetes-controller/temp/NOTES.txt new file mode 100644 index 0000000..f515c84 --- /dev/null +++ b/charts/ocean-kubernetes-controller/temp/NOTES.txt @@ -0,0 +1,22 @@ +1. Get the application URL by running these commands: +{{- if .Values.ingress.enabled }} +{{- range $host := .Values.ingress.hosts }} + {{- range .paths }} + http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }} + {{- end }} +{{- end }} +{{- else if contains "NodePort" .Values.service.type }} + export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "ocean-kubernetes-controller.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo http://$NODE_IP:$NODE_PORT +{{- else if contains "LoadBalancer" .Values.service.type }} + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "ocean-kubernetes-controller.fullname" . }}' + export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "ocean-kubernetes-controller.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") + echo http://$SERVICE_IP:{{ .Values.service.port }} +{{- else if contains "ClusterIP" .Values.service.type }} + export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "ocean-kubernetes-controller.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") + export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") + echo "Visit http://127.0.0.1:8080 to use your application" + kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT +{{- end }} diff --git a/charts/ocean-kubernetes-controller/temp/hpa.yaml b/charts/ocean-kubernetes-controller/temp/hpa.yaml new file mode 100644 index 0000000..8648685 --- /dev/null +++ b/charts/ocean-kubernetes-controller/temp/hpa.yaml @@ -0,0 +1,32 @@ +{{- if .Values.autoscaling.enabled }} +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "ocean-kubernetes-controller.fullname" . }} + labels: + {{- include "ocean-kubernetes-controller.labels" . | nindent 4 }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "ocean-kubernetes-controller.fullname" . }} + minReplicas: {{ .Values.autoscaling.minReplicas }} + maxReplicas: {{ .Values.autoscaling.maxReplicas }} + metrics: + {{- if .Values.autoscaling.targetCPUUtilizationPercentage }} + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} + {{- end }} + {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} + {{- end }} +{{- end }} diff --git a/charts/ocean-kubernetes-controller/temp/ingress.yaml b/charts/ocean-kubernetes-controller/temp/ingress.yaml new file mode 100644 index 0000000..9fb48c7 --- /dev/null +++ b/charts/ocean-kubernetes-controller/temp/ingress.yaml @@ -0,0 +1,61 @@ +{{- if .Values.ingress.enabled -}} +{{- $fullName := include "ocean-kubernetes-controller.fullname" . -}} +{{- $svcPort := .Values.service.port -}} +{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} + {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} + {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} + {{- end }} +{{- end }} +{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1 +{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1beta1 +{{- else -}} +apiVersion: extensions/v1beta1 +{{- end }} +kind: Ingress +metadata: + name: {{ $fullName }} + labels: + {{- include "ocean-kubernetes-controller.labels" . | nindent 4 }} + {{- with .Values.ingress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} + ingressClassName: {{ .Values.ingress.className }} + {{- end }} + {{- if .Values.ingress.tls }} + tls: + {{- range .Values.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} + {{- end }} + rules: + {{- range .Values.ingress.hosts }} + - host: {{ .host | quote }} + http: + paths: + {{- range .paths }} + - path: {{ .path }} + {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} + pathType: {{ .pathType }} + {{- end }} + backend: + {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} + service: + name: {{ $fullName }} + port: + number: {{ $svcPort }} + {{- else }} + serviceName: {{ $fullName }} + servicePort: {{ $svcPort }} + {{- end }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/ocean-kubernetes-controller/temp/service.yaml b/charts/ocean-kubernetes-controller/temp/service.yaml new file mode 100644 index 0000000..9095a93 --- /dev/null +++ b/charts/ocean-kubernetes-controller/temp/service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "ocean-kubernetes-controller.fullname" . }} + labels: + {{- include "ocean-kubernetes-controller.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + {{- include "ocean-kubernetes-controller.selectorLabels" . | nindent 4 }} diff --git a/charts/ocean-kubernetes-controller/temp/serviceaccount.yaml b/charts/ocean-kubernetes-controller/temp/serviceaccount.yaml new file mode 100644 index 0000000..443b2df --- /dev/null +++ b/charts/ocean-kubernetes-controller/temp/serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "ocean-kubernetes-controller.serviceAccountName" . }} + labels: + {{- include "ocean-kubernetes-controller.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/ocean-kubernetes-controller/templates/_helpers.tpl b/charts/ocean-kubernetes-controller/templates/_helpers.tpl new file mode 100644 index 0000000..0e7eead --- /dev/null +++ b/charts/ocean-kubernetes-controller/templates/_helpers.tpl @@ -0,0 +1,119 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "ocean-kubernetes-controller.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "ocean-kubernetes-controller.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "ocean-kubernetes-controller.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +The image to use +*/}} +{{- define "ocean-kubernetes-controller.image" -}} +{{- printf "%s:%s" .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "ocean-kubernetes-controller.labels" -}} +helm.sh/chart: {{ include "ocean-kubernetes-controller.chart" . }} +{{ include "ocean-kubernetes-controller.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "ocean-kubernetes-controller.selectorLabels" -}} +app.kubernetes.io/name: {{ include "ocean-kubernetes-controller.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +ConfigMap name. +*/}} +{{- define "ocean-kubernetes-controller.configMapName" -}} +{{ default (include "ocean-kubernetes-controller.name" .) .Values.configMap.name }} +{{- end }} + +{{/* +Secret name. +*/}} +{{- define "ocean-kubernetes-controller.secretName" -}} +{{ default (include "ocean-kubernetes-controller.name" .) .Values.secret.name }} +{{- end }} + +{{/* +CA bundle secret name. +*/}} +{{- define "ocean-kubernetes-controller.caBundleSecretName" -}} +{{ default (include "ocean-kubernetes-controller.name" .) .Values.caBundleSecret.name }} +{{- end }} + +{{/* +ClusterRole name. +*/}} +{{- define "ocean-kubernetes-controller.clusterRoleName" -}} +{{ include "ocean-kubernetes-controller.name" . }} +{{- end }} + +{{/* +ClusterRoleBinding name. +*/}} +{{- define "ocean-kubernetes-controller.clusterRoleBindingName" -}} +{{ include "ocean-kubernetes-controller.name" . }} +{{- end }} + +{{/* +Deployment name. +*/}} +{{- define "ocean-kubernetes-controller.deploymentName" -}} +{{ include "ocean-kubernetes-controller.name" . }} +{{- end }} + +{{/* +Job name (ocean-aks-connector). +*/}} +{{- define "ocean-kubernetes-controller.aksConnectorJobName" -}} +{{ default (include "ocean-kubernetes-controller.name" .) .Values.aksConnector.jobName }} +{{- end }} + + +{{/* +Create the name of the service account to use +*/}} +{{- define "ocean-kubernetes-controller.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "ocean-kubernetes-controller.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/charts/ocean-kubernetes-controller/templates/deployment.yaml b/charts/ocean-kubernetes-controller/templates/deployment.yaml new file mode 100644 index 0000000..1fe17a2 --- /dev/null +++ b/charts/ocean-kubernetes-controller/templates/deployment.yaml @@ -0,0 +1,167 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "ocean-kubernetes-controller.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "ocean-kubernetes-controller.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + {{- with .Values.updateStrategy }} + strategy: + {{- toYaml . | nindent 4 }} + {{- end }} + selector: + matchLabels: + {{- include "ocean-kubernetes-controller.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "ocean-kubernetes-controller.selectorLabels" . | nindent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "ocean-kubernetes-controller.serviceAccountName" . }} + {{- with .Values.podSecurityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.priorityClassName }} + priorityClassName: {{ . | quote }} + {{- end }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: {{ include "ocean-kubernetes-controller.image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + {{- range .Values.args }} + - {{ . }} + {{- end }} + env: + - name: SPOTINST_TOKEN + valueFrom: + secretKeyRef: + name: {{ include "ocean-kubernetes-controller.secretName" . }} + key: token + optional: true + - name: SPOTINST_ACCOUNT + valueFrom: + secretKeyRef: + name: {{ include "ocean-kubernetes-controller.secretName" . }} + key: account + optional: true + - name: SPOTINST_TOKEN_LEGACY + valueFrom: + configMapKeyRef: + name: {{ include "ocean-kubernetes-controller.configMapName" . }} + key: spotinst.token + optional: true + - name: SPOTINST_ACCOUNT_LEGACY + valueFrom: + configMapKeyRef: + name: {{ include "ocean-kubernetes-controller.configMapName" . }} + key: spotinst.account + optional: true + - name: CLUSTER_IDENTIFIER + valueFrom: + configMapKeyRef: + name: {{ include "ocean-kubernetes-controller.configMapName" . }} + key: spotinst.cluster-identifier + - name: BASE_SPOTINST_URL + valueFrom: + configMapKeyRef: + name: {{ include "ocean-kubernetes-controller.configMapName" . }} + key: base-url + optional: true + - name: PROXY_URL + valueFrom: + configMapKeyRef: + name: {{ include "ocean-kubernetes-controller.configMapName" . }} + key: proxy-url + optional: true + - name: DISABLE_AUTO_UPDATE + valueFrom: + configMapKeyRef: + name: {{ include "ocean-kubernetes-controller.configMapName" . }} + key: disable-auto-update + optional: true + - name: ENABLE_CSR_APPROVAL + valueFrom: + configMapKeyRef: + name: {{ include "ocean-kubernetes-controller.configMapName" . }} + key: enable-csr-approval + optional: true + - name: USER_ENV_CERTIFICATES + valueFrom: + secretKeyRef: + name: {{ include "ocean-kubernetes-controller.caBundleSecretName" . }} + key: userEnvCertificates.pem + optional: true + - name: POD_ID + valueFrom: + fieldRef: + fieldPath: metadata.uid + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + {{- with .Values.extraEnv }} + {{- toYaml . | nindent 10 }} + {{- end }} + ports: + - name: metrics + containerPort: 9080 + - name: readiness + containerPort: 9081 + {{- with .Values.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 12 }} + {{- end }} + volumeMounts: + {{- with .Values.extraVolumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + resources: + {{- with .Values.resources }} + {{- toYaml . | nindent 12 }} + {{- end }} + volumes: + # TODO: handle caBundleSecret volume + {{- with .Values.extraVolumes }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.topologySpreadConstraints }} + topologySpreadConstraints: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/charts/ocean-kubernetes-controller/templates/tests/test-connection.yaml b/charts/ocean-kubernetes-controller/templates/tests/test-connection.yaml new file mode 100644 index 0000000..a8b2204 --- /dev/null +++ b/charts/ocean-kubernetes-controller/templates/tests/test-connection.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Pod +metadata: + name: "{{ include "ocean-kubernetes-controller.fullname" . }}-test-connection" + labels: + {{- include "ocean-kubernetes-controller.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": test +spec: + containers: + - name: wget + image: busybox + command: ['wget'] + args: ['{{ include "ocean-kubernetes-controller.fullname" . }}:{{ .Values.service.port }}'] + restartPolicy: Never diff --git a/charts/ocean-kubernetes-controller/values.yaml b/charts/ocean-kubernetes-controller/values.yaml new file mode 100644 index 0000000..a1af8f9 --- /dev/null +++ b/charts/ocean-kubernetes-controller/values.yaml @@ -0,0 +1,207 @@ +# Default values for ocean-kubernetes-controller. + +# Spot Configuration. +spotinst: + # -- Spot Token. (Required) + # Ref: https://docs.spot.io/administration/api/create-api-token + token: "" + # -- Spot Account. (Required) + # Ref: https://docs.spot.io/administration/organizations?id=account + account: "" + # -- Unique identifier used by the Ocean Controller to connect (Required) + # between the Ocean backend and the Kubernetes cluster. + # Ref: https://docs.spot.io/ocean/tutorials/spot-kubernetes-controller/ + clusterIdentifier: "" + # -- Base URL. (Optional) + baseUrl: "" + # -- Proxy URL. (Optional) + proxyUrl: "" + # -- Disable auto update. (Optional) + disableAutoUpdate: false + # -- Enable CSR approval. (Optional) + enableCsrApproval: false + +nameOverride: "" +fullnameOverride: "" + +# -- Configure the amount of replicas for the controller (Optional) +replicas: 3 + +priorityClassName: system-cluster-critical + +updateStrategy: {} +# type: RollingUpdate +# rollingUpdate: +# maxSurge: 0 +# maxUnavailable: 1 + +image: + repository: us-docker.pkg.dev/spotit-today/container-labs/spotinst-kubernetes-controller + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: "" + +imagePullSecrets: [] + +serviceAccount: + # Specifies whether a service account should be created + create: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. (Optional) + name: "spotinst-kubernetes-cluster-controller" + +secret: + # -- Controls whether a Secret should be created. (Optional) + create: true + # -- Secret name. (Optional) + name: spotinst-kubernetes-cluster-controller + +# CA bundle. +# Ref: https://kubernetes.io/docs/concepts/configuration/secret/ +caBundleSecret: + create: false + # -- Secret name. (Optional) + name: spotinst-kubernetes-cluster-controller-ca-bundle + +# Config Map. +# Ref: https://kubernetes.io/docs/concepts/configuration/configmap/ +configMap: + create: true + # -- ConfigMap name. (Optional) + name: spotinst-kubernetes-cluster-controller-config + +podAnnotations: {} +podLabels: {} +commonLabels: {} + +# Pod Security Context +# Ref: https://kubernetes.io/docs/concepts/security/pod-security-standards/ +podSecurityContext: + runAsNonRoot: true + runAsUser: 10001 + runAsGroup: 10001 + fsGroup: 10001 + +priorityClassName: system-cluster-critical + +# Container Security Context +securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + capabilities: + drop: + - ALL + +args: [] +# - --test + +extraEnv: [] +# - name: KEY +# value: VALUE + +livenessProbe: + httpGet: + path: /healthz + port: readiness + initialDelaySeconds: 15 + periodSeconds: 20 + +readinessProbe: + httpGet: + path: /readyz + port: readiness + initialDelaySeconds: 5 + periodSeconds: 10 + +service: + type: ClusterIP + port: 80 + +# Controller pod resources. (Optional) +resources: {} + # requests: + # cpu: 100m + # memory: 128Mi + # limits: + # cpu: 100m + # memory: 128Mi + + +nodeSelector: {} + +# -- Tolerations for nodes that have taints on them. (Optional) +# Ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ +tolerations: +- key: node.kubernetes.io/not-ready + effect: NoExecute + operator: Exists + tolerationSeconds: 150 +- key: node.kubernetes.io/unreachable + effect: NoExecute + operator: Exists + tolerationSeconds: 150 +- key: node-role.kubernetes.io/master + operator: Exists +- key: node-role.kubernetes.io/control-plane + operator: Exists + +# Pod scheduling preferences. +# Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity +affinity: + # -- Node affinity. (Optional) + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/os + operator: NotIn + values: + - windows + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + # -- Pod anti-affinity. (Optional) + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 50 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app.kubernetes.io/name + operator: In + values: + - spotinst-kubernetes-cluster-controller + topologyKey: kubernetes.io/hostname + +extraVolumeMounts: [] + +extraVolumes: [] + +topologySpreadConstraints: [] + +# Annotations to add to the deployment +deploymentAnnotations: {} + +schedulerName: "" + +# Metrics Server configuration. +metrics-server: + # -- Specifies whether the metrics-server chart should be deployed. (Optional) + deployChart: true + + # Overrides the image + image: + repository: registry.k8s.io/metrics-server/metrics-server + tag: "" + pullPolicy: IfNotPresent + + # -- Arguments to pass to metrics-server on start up. (Optional) + args: + - --logtostderr + # enable this if you have self-signed certificates, see: https://github.com/kubernetes-incubator/metrics-server + # - --kubelet-insecure-tls From ce024c1fe8ddc00d35764c9d21cbdb96c67438dc Mon Sep 17 00:00:00 2001 From: Roi Kramer Date: Wed, 20 Sep 2023 11:28:50 +0300 Subject: [PATCH 2/8] wip --- .../temp/NOTES.txt | 22 ---- .../ocean-kubernetes-controller/temp/hpa.yaml | 32 ----- .../temp/ingress.yaml | 61 --------- .../temp/service.yaml | 15 --- .../templates/clusterrole.yaml | 120 ++++++++++++++++++ .../templates/clusterrolebinding.yaml | 14 ++ .../templates/configmap.yaml | 14 ++ .../templates/deployment.yaml | 44 ++++++- .../templates/secret.yaml | 12 ++ .../{temp => templates}/serviceaccount.yaml | 0 .../templates/tests/test-connection.yaml | 15 --- .../ocean-kubernetes-controller/values.yaml | 68 ++++------ 12 files changed, 225 insertions(+), 192 deletions(-) delete mode 100644 charts/ocean-kubernetes-controller/temp/NOTES.txt delete mode 100644 charts/ocean-kubernetes-controller/temp/hpa.yaml delete mode 100644 charts/ocean-kubernetes-controller/temp/ingress.yaml delete mode 100644 charts/ocean-kubernetes-controller/temp/service.yaml create mode 100644 charts/ocean-kubernetes-controller/templates/clusterrole.yaml create mode 100644 charts/ocean-kubernetes-controller/templates/clusterrolebinding.yaml create mode 100644 charts/ocean-kubernetes-controller/templates/configmap.yaml create mode 100644 charts/ocean-kubernetes-controller/templates/secret.yaml rename charts/ocean-kubernetes-controller/{temp => templates}/serviceaccount.yaml (100%) delete mode 100644 charts/ocean-kubernetes-controller/templates/tests/test-connection.yaml diff --git a/charts/ocean-kubernetes-controller/temp/NOTES.txt b/charts/ocean-kubernetes-controller/temp/NOTES.txt deleted file mode 100644 index f515c84..0000000 --- a/charts/ocean-kubernetes-controller/temp/NOTES.txt +++ /dev/null @@ -1,22 +0,0 @@ -1. Get the application URL by running these commands: -{{- if .Values.ingress.enabled }} -{{- range $host := .Values.ingress.hosts }} - {{- range .paths }} - http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }} - {{- end }} -{{- end }} -{{- else if contains "NodePort" .Values.service.type }} - export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "ocean-kubernetes-controller.fullname" . }}) - export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") - echo http://$NODE_IP:$NODE_PORT -{{- else if contains "LoadBalancer" .Values.service.type }} - NOTE: It may take a few minutes for the LoadBalancer IP to be available. - You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "ocean-kubernetes-controller.fullname" . }}' - export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "ocean-kubernetes-controller.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") - echo http://$SERVICE_IP:{{ .Values.service.port }} -{{- else if contains "ClusterIP" .Values.service.type }} - export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "ocean-kubernetes-controller.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") - export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") - echo "Visit http://127.0.0.1:8080 to use your application" - kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT -{{- end }} diff --git a/charts/ocean-kubernetes-controller/temp/hpa.yaml b/charts/ocean-kubernetes-controller/temp/hpa.yaml deleted file mode 100644 index 8648685..0000000 --- a/charts/ocean-kubernetes-controller/temp/hpa.yaml +++ /dev/null @@ -1,32 +0,0 @@ -{{- if .Values.autoscaling.enabled }} -apiVersion: autoscaling/v2 -kind: HorizontalPodAutoscaler -metadata: - name: {{ include "ocean-kubernetes-controller.fullname" . }} - labels: - {{- include "ocean-kubernetes-controller.labels" . | nindent 4 }} -spec: - scaleTargetRef: - apiVersion: apps/v1 - kind: Deployment - name: {{ include "ocean-kubernetes-controller.fullname" . }} - minReplicas: {{ .Values.autoscaling.minReplicas }} - maxReplicas: {{ .Values.autoscaling.maxReplicas }} - metrics: - {{- if .Values.autoscaling.targetCPUUtilizationPercentage }} - - type: Resource - resource: - name: cpu - target: - type: Utilization - averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} - {{- end }} - {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} - - type: Resource - resource: - name: memory - target: - type: Utilization - averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} - {{- end }} -{{- end }} diff --git a/charts/ocean-kubernetes-controller/temp/ingress.yaml b/charts/ocean-kubernetes-controller/temp/ingress.yaml deleted file mode 100644 index 9fb48c7..0000000 --- a/charts/ocean-kubernetes-controller/temp/ingress.yaml +++ /dev/null @@ -1,61 +0,0 @@ -{{- if .Values.ingress.enabled -}} -{{- $fullName := include "ocean-kubernetes-controller.fullname" . -}} -{{- $svcPort := .Values.service.port -}} -{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} - {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} - {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} - {{- end }} -{{- end }} -{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} -apiVersion: networking.k8s.io/v1 -{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} -apiVersion: networking.k8s.io/v1beta1 -{{- else -}} -apiVersion: extensions/v1beta1 -{{- end }} -kind: Ingress -metadata: - name: {{ $fullName }} - labels: - {{- include "ocean-kubernetes-controller.labels" . | nindent 4 }} - {{- with .Values.ingress.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -spec: - {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} - ingressClassName: {{ .Values.ingress.className }} - {{- end }} - {{- if .Values.ingress.tls }} - tls: - {{- range .Values.ingress.tls }} - - hosts: - {{- range .hosts }} - - {{ . | quote }} - {{- end }} - secretName: {{ .secretName }} - {{- end }} - {{- end }} - rules: - {{- range .Values.ingress.hosts }} - - host: {{ .host | quote }} - http: - paths: - {{- range .paths }} - - path: {{ .path }} - {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} - pathType: {{ .pathType }} - {{- end }} - backend: - {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} - service: - name: {{ $fullName }} - port: - number: {{ $svcPort }} - {{- else }} - serviceName: {{ $fullName }} - servicePort: {{ $svcPort }} - {{- end }} - {{- end }} - {{- end }} -{{- end }} diff --git a/charts/ocean-kubernetes-controller/temp/service.yaml b/charts/ocean-kubernetes-controller/temp/service.yaml deleted file mode 100644 index 9095a93..0000000 --- a/charts/ocean-kubernetes-controller/temp/service.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: {{ include "ocean-kubernetes-controller.fullname" . }} - labels: - {{- include "ocean-kubernetes-controller.labels" . | nindent 4 }} -spec: - type: {{ .Values.service.type }} - ports: - - port: {{ .Values.service.port }} - targetPort: http - protocol: TCP - name: http - selector: - {{- include "ocean-kubernetes-controller.selectorLabels" . | nindent 4 }} diff --git a/charts/ocean-kubernetes-controller/templates/clusterrole.yaml b/charts/ocean-kubernetes-controller/templates/clusterrole.yaml new file mode 100644 index 0000000..3cfb62f --- /dev/null +++ b/charts/ocean-kubernetes-controller/templates/clusterrole.yaml @@ -0,0 +1,120 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "ocean-kubernetes-controller.fullname" . }} + labels: + {{- include "ocean-kubernetes-controller.labels" . | nindent 4 }} +rules: +# --------------------------------------------------------------------------- +# feature: ocean/readonly +# --------------------------------------------------------------------------- +- apiGroups: [ "" ] + resources: [ "pods", "nodes", "services", "namespaces", "replicationcontrollers", "limitranges", "events", "persistentvolumes", "persistentvolumeclaims" ] + verbs: [ "get", "list", "watch" ] +- apiGroups: [ "apps" ] + resources: [ "deployments", "daemonsets", "statefulsets", "replicasets" ] + verbs: [ "get", "list", "watch" ] +- apiGroups: [ "storage.k8s.io" ] + resources: [ "storageclasses" ] + verbs: [ "get", "list", "watch" ] +- apiGroups: [ "batch" ] + resources: [ "jobs", "cronjobs" ] + verbs: [ "get", "list", "watch" ] +- apiGroups: [ "policy" ] + resources: [ "poddisruptionbudgets" ] + verbs: [ "get", "list", "watch" ] +- apiGroups: [ "metrics.k8s.io" ] + resources: [ "pods" ] + verbs: [ "get", "list", "watch" ] +- apiGroups: [ "autoscaling" ] + resources: [ "horizontalpodautoscalers" ] + verbs: [ "get", "list", "watch" ] +- apiGroups: [ "apiextensions.k8s.io" ] + resources: [ "customresourcedefinitions" ] + verbs: [ "get", "list", "watch" ] +- apiGroups: [ "node.k8s.io" ] + resources: [ "runtimeclasses" ] + verbs: [ "get", "list", "watch" ] +- nonResourceURLs: [ "/version/", "/version" ] + verbs: [ "get" ] +# --------------------------------------------------------------------------- +# feature: ocean/draining +# --------------------------------------------------------------------------- +- apiGroups: [""] + resources: ["nodes"] + verbs: ["patch", "update"] +- apiGroups: [""] + resources: ["pods"] + verbs: ["delete"] +- apiGroups: [""] + resources: ["pods/eviction"] + verbs: ["create"] +# --------------------------------------------------------------------------- +# feature: ocean/cleanup +# --------------------------------------------------------------------------- +- apiGroups: [""] + resources: ["nodes"] + verbs: ["delete"] +# --------------------------------------------------------------------------- +# feature: ocean/csr-approval +# --------------------------------------------------------------------------- +- apiGroups: ["certificates.k8s.io"] + resources: ["certificatesigningrequests"] + verbs: ["get", "list", "delete", "create"] +- apiGroups: ["certificates.k8s.io"] + resources: ["certificatesigningrequests/approval"] + verbs: ["patch", "update"] +- apiGroups: ["certificates.k8s.io"] + resources: ["signers"] + resourceNames: ["kubernetes.io/kubelet-serving", "kubernetes.io/kube-apiserver-client-kubelet"] + verbs: ["approve"] +# --------------------------------------------------------------------------- +# feature: ocean/auto-update +# --------------------------------------------------------------------------- +- apiGroups: ["rbac.authorization.k8s.io"] + resources: ["clusterroles"] + resourceNames: ["spotinst-kubernetes-cluster-controller"] + verbs: ["patch", "update", "escalate"] +- apiGroups: ["apps"] + resources: ["deployments"] + resourceNames: ["spotinst-kubernetes-cluster-controller"] + verbs: ["patch", "update"] +# --------------------------------------------------------------------------- +# feature: ocean/apply +# --------------------------------------------------------------------------- +- apiGroups: ["apps"] + resources: ["deployments", "daemonsets"] + verbs: ["get", "list", "patch", "update", "create", "delete"] +- apiGroups: ["extensions"] + resources: ["daemonsets"] + verbs: ["get", "list", "patch", "update", "create", "delete"] +- apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "patch", "update", "create", "delete"] +- apiGroups: ["batch"] + resources: ["jobs"] + verbs: ["get", "list", "patch", "update", "create", "delete"] +# --------------------------------------------------------------------------- +# feature: wave +# --------------------------------------------------------------------------- +- apiGroups: ["sparkoperator.k8s.io"] + resources: ["sparkapplications", "scheduledsparkapplications"] + verbs: ["get", "list", "patch", "update", "create", "delete"] +- apiGroups: ["wave.spot.io"] + resources: ["sparkapplications", "wavecomponents", "waveenvironments"] + verbs: ["get", "list"] +- apiGroups: ["bigdata.spot.io"] + resources: ["bigdataenvironments"] + verbs: ["get", "list", "patch", "update", "create", "delete"] +# --------------------------------------------------------------------------- +# feature: controller/leader-election (high-availability) +# --------------------------------------------------------------------------- +- apiGroups: [ "coordination.k8s.io" ] + resources: [ "leases" ] + verbs: [ "get","list","patch","update","create","delete" ] +# --------------------------------------------------------------------------- +# feature: controller/report-events +# --------------------------------------------------------------------------- +- apiGroups: [ "" ] + resources: [ "events" ] + verbs: [ "create" ] \ No newline at end of file diff --git a/charts/ocean-kubernetes-controller/templates/clusterrolebinding.yaml b/charts/ocean-kubernetes-controller/templates/clusterrolebinding.yaml new file mode 100644 index 0000000..a73fd74 --- /dev/null +++ b/charts/ocean-kubernetes-controller/templates/clusterrolebinding.yaml @@ -0,0 +1,14 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "ocean-kubernetes-controller.fullname" . }} + labels: + {{- include "ocean-kubernetes-controller.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "ocean-kubernetes-controller.fullname" . }} +subjects: +- kind: ServiceAccount + name: {{ include "ocean-kubernetes-controller.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} diff --git a/charts/ocean-kubernetes-controller/templates/configmap.yaml b/charts/ocean-kubernetes-controller/templates/configmap.yaml new file mode 100644 index 0000000..1c8cf09 --- /dev/null +++ b/charts/ocean-kubernetes-controller/templates/configmap.yaml @@ -0,0 +1,14 @@ +{{- if .Values.configMap.create }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "ocean-kubernetes-controller.configMapName" . }} + labels: + {{- include "ocean-kubernetes-controller.labels" . | nindent 4 }} +data: + spotinst.cluster-identifier: {{ required "`spotinst.clusterIdentifier` must be specified" .Values.spotinst.clusterIdentifier }} + base-url: {{ default "" .Values.spotinst.baseUrl | quote }} + proxy-url: {{ default "" .Values.spotinst.proxyUrl | quote }} + disable-auto-update: {{ default "false" .Values.spotinst.disableAutoUpdate | quote }} + enable-csr-approval: {{ default "false" .Values.spotinst.enableCsrApproval | quote }} +{{- end }} diff --git a/charts/ocean-kubernetes-controller/templates/deployment.yaml b/charts/ocean-kubernetes-controller/templates/deployment.yaml index 1fe17a2..62c788e 100644 --- a/charts/ocean-kubernetes-controller/templates/deployment.yaml +++ b/charts/ocean-kubernetes-controller/templates/deployment.yaml @@ -153,15 +153,57 @@ spec: nodeSelector: {{- toYaml . | nindent 8 }} {{- end }} + {{- if kindIs "invalid" .Values.affinity }} + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/os + operator: NotIn + values: + - windows + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + {{- else }} {{- with .Values.affinity }} affinity: {{- toYaml . | nindent 8 }} {{- end }} - {{- with .Values.tolerations }} + {{- end }} + {{- if kindIs "invalid" .Values.tolerations }} tolerations: + - key: node.kubernetes.io/not-ready + effect: NoExecute + operator: Exists + tolerationSeconds: 150 + - key: node.kubernetes.io/unreachable + effect: NoExecute + operator: Exists + tolerationSeconds: 150 + - key: node-role.kubernetes.io/master + operator: Exists + - key: node-role.kubernetes.io/control-plane + operator: Exists + {{- else }} + {{- with .Values.tolerations }} + tolerations: {{- toYaml . | nindent 8 }} {{- end }} + {{- end }} + {{- if kindIs "invalid" .Values.topologySpreadConstraints }} + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + labelSelector: + {{- include "ocean-kubernetes-controller.selectorLabels" . | nindent 6 }} + {{- else }} {{- with .Values.topologySpreadConstraints }} topologySpreadConstraints: {{- toYaml . | nindent 8 }} {{- end }} + {{- end }} diff --git a/charts/ocean-kubernetes-controller/templates/secret.yaml b/charts/ocean-kubernetes-controller/templates/secret.yaml new file mode 100644 index 0000000..38f01b0 --- /dev/null +++ b/charts/ocean-kubernetes-controller/templates/secret.yaml @@ -0,0 +1,12 @@ +{{- if and .Values.secret.create }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "ocean-kubernetes-controller.secretName" . }} + labels: + {{- include "ocean-kubernetes-controller.labels" . | nindent 4 }} +type: Opaque +data: + token: {{ required "`spotinst.token` must be specified" .Values.spotinst.token | quote | b64enc }} + account: {{ required "`spotinst.account` must be specified" .Values.spotinst.account | quote | b64enc }} +{{- end }} diff --git a/charts/ocean-kubernetes-controller/temp/serviceaccount.yaml b/charts/ocean-kubernetes-controller/templates/serviceaccount.yaml similarity index 100% rename from charts/ocean-kubernetes-controller/temp/serviceaccount.yaml rename to charts/ocean-kubernetes-controller/templates/serviceaccount.yaml diff --git a/charts/ocean-kubernetes-controller/templates/tests/test-connection.yaml b/charts/ocean-kubernetes-controller/templates/tests/test-connection.yaml deleted file mode 100644 index a8b2204..0000000 --- a/charts/ocean-kubernetes-controller/templates/tests/test-connection.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: "{{ include "ocean-kubernetes-controller.fullname" . }}-test-connection" - labels: - {{- include "ocean-kubernetes-controller.labels" . | nindent 4 }} - annotations: - "helm.sh/hook": test -spec: - containers: - - name: wget - image: busybox - command: ['wget'] - args: ['{{ include "ocean-kubernetes-controller.fullname" . }}:{{ .Values.service.port }}'] - restartPolicy: Never diff --git a/charts/ocean-kubernetes-controller/values.yaml b/charts/ocean-kubernetes-controller/values.yaml index a1af8f9..983336c 100644 --- a/charts/ocean-kubernetes-controller/values.yaml +++ b/charts/ocean-kubernetes-controller/values.yaml @@ -115,10 +115,6 @@ readinessProbe: initialDelaySeconds: 5 periodSeconds: 10 -service: - type: ClusterIP - port: 80 - # Controller pod resources. (Optional) resources: {} # requests: @@ -134,61 +130,41 @@ nodeSelector: {} # -- Tolerations for nodes that have taints on them. (Optional) # Ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ tolerations: -- key: node.kubernetes.io/not-ready - effect: NoExecute - operator: Exists - tolerationSeconds: 150 -- key: node.kubernetes.io/unreachable - effect: NoExecute - operator: Exists - tolerationSeconds: 150 -- key: node-role.kubernetes.io/master - operator: Exists -- key: node-role.kubernetes.io/control-plane - operator: Exists +# - key: node.kubernetes.io/not-ready +# effect: NoExecute +# operator: Exists +# tolerationSeconds: 150 # Pod scheduling preferences. # Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity affinity: - # -- Node affinity. (Optional) - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: kubernetes.io/os - operator: NotIn - values: - - windows - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 100 - preference: - matchExpressions: - - key: node-role.kubernetes.io/master - operator: Exists - # -- Pod anti-affinity. (Optional) - podAntiAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 50 - podAffinityTerm: - labelSelector: - matchExpressions: - - key: app.kubernetes.io/name - operator: In - values: - - spotinst-kubernetes-cluster-controller - topologyKey: kubernetes.io/hostname +# podAntiAffinity: +# preferredDuringSchedulingIgnoredDuringExecution: +# - weight: 50 +# podAffinityTerm: +# labelSelector: +# matchExpressions: +# - key: app.kubernetes.io/name +# operator: In +# values: +# - spotinst-kubernetes-cluster-controller +# topologyKey: kubernetes.io/hostname + +topologySpreadConstraints: +# - maxSkew: 1 +# topologyKey: kubernetes.io/hostname +# labelSelector: +# app: test extraVolumeMounts: [] extraVolumes: [] -topologySpreadConstraints: [] +schedulerName: "" # Annotations to add to the deployment deploymentAnnotations: {} -schedulerName: "" - # Metrics Server configuration. metrics-server: # -- Specifies whether the metrics-server chart should be deployed. (Optional) From c15d60576435702298fa41e750f31c1143a33c62 Mon Sep 17 00:00:00 2001 From: Roi Kramer Date: Wed, 20 Sep 2023 18:44:33 +0300 Subject: [PATCH 3/8] added check for metrics server --- charts/ocean-kubernetes-controller/Chart.yaml | 2 +- .../templates/_helpers.tpl | 63 ++++++++++++------- .../templates/deployment.yaml | 25 +++++--- .../ocean-kubernetes-controller/values.yaml | 34 +++++----- 4 files changed, 75 insertions(+), 49 deletions(-) diff --git a/charts/ocean-kubernetes-controller/Chart.yaml b/charts/ocean-kubernetes-controller/Chart.yaml index 79b5ecf..eb2fbd6 100644 --- a/charts/ocean-kubernetes-controller/Chart.yaml +++ b/charts/ocean-kubernetes-controller/Chart.yaml @@ -16,4 +16,4 @@ dependencies: - name: metrics-server version: 3.11.0 repository: https://kubernetes-sigs.github.io/metrics-server - condition: metrics-server.deployChart \ No newline at end of file + condition: metrics-server.enabled \ No newline at end of file diff --git a/charts/ocean-kubernetes-controller/templates/_helpers.tpl b/charts/ocean-kubernetes-controller/templates/_helpers.tpl index 0e7eead..de86c68 100644 --- a/charts/ocean-kubernetes-controller/templates/_helpers.tpl +++ b/charts/ocean-kubernetes-controller/templates/_helpers.tpl @@ -61,54 +61,39 @@ app.kubernetes.io/instance: {{ .Release.Name }} ConfigMap name. */}} {{- define "ocean-kubernetes-controller.configMapName" -}} -{{ default (include "ocean-kubernetes-controller.name" .) .Values.configMap.name }} +{{ default (include "ocean-kubernetes-controller.fullname" .) .Values.configMap.name }} {{- end }} {{/* Secret name. */}} {{- define "ocean-kubernetes-controller.secretName" -}} -{{ default (include "ocean-kubernetes-controller.name" .) .Values.secret.name }} +{{ default (include "ocean-kubernetes-controller.fullname" .) .Values.secret.name }} {{- end }} {{/* CA bundle secret name. */}} {{- define "ocean-kubernetes-controller.caBundleSecretName" -}} -{{ default (include "ocean-kubernetes-controller.name" .) .Values.caBundleSecret.name }} +{{ default (include "ocean-kubernetes-controller.fullname" .) .Values.caBundleSecret.name }} {{- end }} {{/* ClusterRole name. */}} {{- define "ocean-kubernetes-controller.clusterRoleName" -}} -{{ include "ocean-kubernetes-controller.name" . }} +{{ include "ocean-kubernetes-controller.fullname" . }} {{- end }} {{/* ClusterRoleBinding name. */}} {{- define "ocean-kubernetes-controller.clusterRoleBindingName" -}} -{{ include "ocean-kubernetes-controller.name" . }} +{{ include "ocean-kubernetes-controller.fullname" . }} {{- end }} {{/* -Deployment name. -*/}} -{{- define "ocean-kubernetes-controller.deploymentName" -}} -{{ include "ocean-kubernetes-controller.name" . }} -{{- end }} - -{{/* -Job name (ocean-aks-connector). -*/}} -{{- define "ocean-kubernetes-controller.aksConnectorJobName" -}} -{{ default (include "ocean-kubernetes-controller.name" .) .Values.aksConnector.jobName }} -{{- end }} - - -{{/* -Create the name of the service account to use +Create the name of the service-account to use */}} {{- define "ocean-kubernetes-controller.serviceAccountName" -}} {{- if .Values.serviceAccount.create }} @@ -117,3 +102,39 @@ Create the name of the service account to use {{- default "default" .Values.serviceAccount.name }} {{- end }} {{- end }} + +{{/* +Figure out if we should deploy metrics server. We are checking: +- if 'metrics-server.deployChart' is true: + - try to fetch the 'v1beta1.metrics.k8s.io' APIService + - if it exists: + - check for it's helm annotations to see if it was installed as part of the + same release we are installing now (release name and namespace annotations). + - if it's not the same release -> fail +*/}} +{{- define "ocean-kubernetes-controller.deployMetricsServer" }} +{{- if (index .Values "metrics-server" "deployChart") }} +{{- $apiService := lookup "apiregistration.k8s.io/v1" "APIService" "" "v1beta1.metrics.k8s.io" }} +{{- $releaseName := .Release.Name }} +{{- $releaseNamespace := .Release.Namespace }} +{{- if $apiService -}} +{{- with $apiService }} +{{- if (or + (not .metadata.annotations) + (or + (ne + $releaseName + (index .metadata.annotations "meta.helm.sh/release-name") + ) + (ne + $releaseNamespace + (index .metadata.annotations "meta.helm.sh/release-namespace") + ) + )) +}} +{{- fail "\nThe value: 'metrics-server.deployChart' was set to 'true' but we found another installation of metrics-server in your cluster.\nYou must use:\n --set metrics-server.deployChart=false" }} +{{- end }} +{{- end }} +{{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/ocean-kubernetes-controller/templates/deployment.yaml b/charts/ocean-kubernetes-controller/templates/deployment.yaml index 62c788e..0636f61 100644 --- a/charts/ocean-kubernetes-controller/templates/deployment.yaml +++ b/charts/ocean-kubernetes-controller/templates/deployment.yaml @@ -1,3 +1,4 @@ +{{ include "ocean-kubernetes-controller.deployMetricsServer" . }} apiVersion: apps/v1 kind: Deployment metadata: @@ -16,10 +17,12 @@ spec: {{- include "ocean-kubernetes-controller.selectorLabels" . | nindent 6 }} template: metadata: - {{- with .Values.podAnnotations }} annotations: - {{- toYaml . | nindent 8 }} - {{- end }} + # This will restart the deployment in case of configmap changes + checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} + {{- with .Values.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} labels: {{- include "ocean-kubernetes-controller.selectorLabels" . | nindent 8 }} {{- with .Values.podLabels }} @@ -163,12 +166,12 @@ spec: operator: NotIn values: - windows - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 100 - preference: - matchExpressions: - - key: node-role.kubernetes.io/master - operator: Exists + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists {{- else }} {{- with .Values.affinity }} affinity: @@ -199,8 +202,10 @@ spec: topologySpreadConstraints: - maxSkew: 1 topologyKey: kubernetes.io/hostname + whenUnsatisfiable: ScheduleAnyway labelSelector: - {{- include "ocean-kubernetes-controller.selectorLabels" . | nindent 6 }} + matchLabels: + {{- include "ocean-kubernetes-controller.selectorLabels" . | nindent 12 }} {{- else }} {{- with .Values.topologySpreadConstraints }} topologySpreadConstraints: diff --git a/charts/ocean-kubernetes-controller/values.yaml b/charts/ocean-kubernetes-controller/values.yaml index 983336c..42ae504 100644 --- a/charts/ocean-kubernetes-controller/values.yaml +++ b/charts/ocean-kubernetes-controller/values.yaml @@ -1,5 +1,8 @@ # Default values for ocean-kubernetes-controller. +nameOverride: "" +fullnameOverride: "" + # Spot Configuration. spotinst: # -- Spot Token. (Required) @@ -21,20 +24,9 @@ spotinst: # -- Enable CSR approval. (Optional) enableCsrApproval: false -nameOverride: "" -fullnameOverride: "" - # -- Configure the amount of replicas for the controller (Optional) replicas: 3 -priorityClassName: system-cluster-critical - -updateStrategy: {} -# type: RollingUpdate -# rollingUpdate: -# maxSurge: 0 -# maxUnavailable: 1 - image: repository: us-docker.pkg.dev/spotit-today/container-labs/spotinst-kubernetes-controller pullPolicy: IfNotPresent @@ -49,27 +41,26 @@ serviceAccount: # Annotations to add to the service account annotations: {} # The name of the service account to use. (Optional) - name: "spotinst-kubernetes-cluster-controller" + name: "" secret: # -- Controls whether a Secret should be created. (Optional) create: true # -- Secret name. (Optional) - name: spotinst-kubernetes-cluster-controller + name: "" # CA bundle. # Ref: https://kubernetes.io/docs/concepts/configuration/secret/ caBundleSecret: - create: false # -- Secret name. (Optional) - name: spotinst-kubernetes-cluster-controller-ca-bundle + name: "" # Config Map. # Ref: https://kubernetes.io/docs/concepts/configuration/configmap/ configMap: create: true # -- ConfigMap name. (Optional) - name: spotinst-kubernetes-cluster-controller-config + name: "" podAnnotations: {} podLabels: {} @@ -94,6 +85,8 @@ securityContext: drop: - ALL +priorityClassName: system-cluster-critical + args: [] # - --test @@ -124,7 +117,6 @@ resources: {} # cpu: 100m # memory: 128Mi - nodeSelector: {} # -- Tolerations for nodes that have taints on them. (Optional) @@ -153,6 +145,7 @@ affinity: topologySpreadConstraints: # - maxSkew: 1 # topologyKey: kubernetes.io/hostname +# whenUnsatisfiable: ScheduleAnyway # labelSelector: # app: test @@ -165,6 +158,13 @@ schedulerName: "" # Annotations to add to the deployment deploymentAnnotations: {} +# Deployment update strategy +updateStrategy: {} +# type: RollingUpdate +# rollingUpdate: +# maxSurge: 0 +# maxUnavailable: 1 + # Metrics Server configuration. metrics-server: # -- Specifies whether the metrics-server chart should be deployed. (Optional) From 8097c2b3005e8d4e280a908445b93d432ab4509a Mon Sep 17 00:00:00 2001 From: Roi Kramer Date: Wed, 20 Sep 2023 19:12:22 +0300 Subject: [PATCH 4/8] fixed leader election --- .../templates/configmap.yaml | 1 + .../templates/deployment.yaml | 12 ++++++++++-- .../templates/secret.yaml | 4 ++-- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/charts/ocean-kubernetes-controller/templates/configmap.yaml b/charts/ocean-kubernetes-controller/templates/configmap.yaml index 1c8cf09..5979fef 100644 --- a/charts/ocean-kubernetes-controller/templates/configmap.yaml +++ b/charts/ocean-kubernetes-controller/templates/configmap.yaml @@ -9,6 +9,7 @@ data: spotinst.cluster-identifier: {{ required "`spotinst.clusterIdentifier` must be specified" .Values.spotinst.clusterIdentifier }} base-url: {{ default "" .Values.spotinst.baseUrl | quote }} proxy-url: {{ default "" .Values.spotinst.proxyUrl | quote }} + leader-election: {{ gt (int .Values.replicas) 1 | quote }} disable-auto-update: {{ default "false" .Values.spotinst.disableAutoUpdate | quote }} enable-csr-approval: {{ default "false" .Values.spotinst.enableCsrApproval | quote }} {{- end }} diff --git a/charts/ocean-kubernetes-controller/templates/deployment.yaml b/charts/ocean-kubernetes-controller/templates/deployment.yaml index 0636f61..730572d 100644 --- a/charts/ocean-kubernetes-controller/templates/deployment.yaml +++ b/charts/ocean-kubernetes-controller/templates/deployment.yaml @@ -7,7 +7,7 @@ metadata: labels: {{- include "ocean-kubernetes-controller.labels" . | nindent 4 }} spec: - replicas: {{ .Values.replicaCount }} + replicas: {{ .Values.replicas }} {{- with .Values.updateStrategy }} strategy: {{- toYaml . | nindent 4 }} @@ -18,8 +18,10 @@ spec: template: metadata: annotations: - # This will restart the deployment in case of configmap changes + # This will restart the deployment in case of configmap/secret changes checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} + checksum/secret: {{ include (print $.Template.BasePath "/secret.yaml") . | sha256sum }} + {{- with .Values.podAnnotations }} {{- toYaml . | nindent 8 }} {{- end }} @@ -70,6 +72,12 @@ spec: name: {{ include "ocean-kubernetes-controller.configMapName" . }} key: spotinst.token optional: true + - name: SPOTINST_LEADER_ELECTION_ENABLED + valueFrom: + configMapKeyRef: + name: {{ include "ocean-kubernetes-controller.configMapName" . }} + key: leader-election + optional: true - name: SPOTINST_ACCOUNT_LEGACY valueFrom: configMapKeyRef: diff --git a/charts/ocean-kubernetes-controller/templates/secret.yaml b/charts/ocean-kubernetes-controller/templates/secret.yaml index 38f01b0..54d3405 100644 --- a/charts/ocean-kubernetes-controller/templates/secret.yaml +++ b/charts/ocean-kubernetes-controller/templates/secret.yaml @@ -7,6 +7,6 @@ metadata: {{- include "ocean-kubernetes-controller.labels" . | nindent 4 }} type: Opaque data: - token: {{ required "`spotinst.token` must be specified" .Values.spotinst.token | quote | b64enc }} - account: {{ required "`spotinst.account` must be specified" .Values.spotinst.account | quote | b64enc }} + token: {{ required "`spotinst.token` must be specified" .Values.spotinst.token | b64enc }} + account: {{ required "`spotinst.account` must be specified" .Values.spotinst.account | b64enc }} {{- end }} From 17c17155db80863896424e84fb8f93accbb0120d Mon Sep 17 00:00:00 2001 From: Roi Kramer Date: Wed, 20 Sep 2023 19:26:39 +0300 Subject: [PATCH 5/8] bump appVersion 2.0.17 --- charts/ocean-kubernetes-controller/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/ocean-kubernetes-controller/Chart.yaml b/charts/ocean-kubernetes-controller/Chart.yaml index eb2fbd6..c030706 100644 --- a/charts/ocean-kubernetes-controller/Chart.yaml +++ b/charts/ocean-kubernetes-controller/Chart.yaml @@ -3,7 +3,7 @@ name: ocean-kubernetes-controller description: A Helm chart for Ocean Controller type: application version: 0.1.0 -appVersion: "2.0.16" +appVersion: 2.0.17 maintainers: - name: spotinst email: ng-spot-info@netapp.com From d3a0c1a93249b4cc9aaa32c92d6749ff5d83fd4e Mon Sep 17 00:00:00 2001 From: Roi Kramer Date: Thu, 21 Sep 2023 10:35:34 +0300 Subject: [PATCH 6/8] fix ca bundle secret default name --- charts/ocean-kubernetes-controller/templates/_helpers.tpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/ocean-kubernetes-controller/templates/_helpers.tpl b/charts/ocean-kubernetes-controller/templates/_helpers.tpl index de86c68..d1963e6 100644 --- a/charts/ocean-kubernetes-controller/templates/_helpers.tpl +++ b/charts/ocean-kubernetes-controller/templates/_helpers.tpl @@ -75,7 +75,7 @@ Secret name. CA bundle secret name. */}} {{- define "ocean-kubernetes-controller.caBundleSecretName" -}} -{{ default (include "ocean-kubernetes-controller.fullname" .) .Values.caBundleSecret.name }} +{{ default (printf "%s-ca-bundle" (include "ocean-kubernetes-controller.fullname" .)) .Values.caBundleSecret.name }} {{- end }} {{/* From e5d21e3ec1e2d6c1b0e08765a5717477b23f73b8 Mon Sep 17 00:00:00 2001 From: Roi Kramer Date: Thu, 21 Sep 2023 16:24:38 +0300 Subject: [PATCH 7/8] wip --- charts/ocean-kubernetes-controller/Chart.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/ocean-kubernetes-controller/Chart.lock b/charts/ocean-kubernetes-controller/Chart.lock index 246508a..6429d10 100644 --- a/charts/ocean-kubernetes-controller/Chart.lock +++ b/charts/ocean-kubernetes-controller/Chart.lock @@ -2,5 +2,5 @@ dependencies: - name: metrics-server repository: https://kubernetes-sigs.github.io/metrics-server version: 3.11.0 -digest: sha256:472915f28d2f1016d5e51ec6c5173c3c01856132a1012dc951242452533b99a9 -generated: "2023-09-19T19:33:55.787057+03:00" +digest: sha256:d72c6e2556ad01652833d9a81cd6ec626611244912a878d32d9ed58203d831bb +generated: "2023-09-21T16:24:22.598098+03:00" From 60ef92b828376d93ce19a28b0f5d908a1d21c6ba Mon Sep 17 00:00:00 2001 From: Roi Kramer Date: Sun, 24 Sep 2023 09:49:15 +0300 Subject: [PATCH 8/8] test --- .github/ct.yaml | 2 ++ charts/ocean-kubernetes-controller/Chart.yaml | 5 ++++- .../ocean-kubernetes-controller/templates/clusterrole.yaml | 3 --- charts/ocean-kubernetes-controller/templates/deployment.yaml | 1 - charts/ocean-kubernetes-controller/values.yaml | 2 -- 5 files changed, 6 insertions(+), 7 deletions(-) diff --git a/.github/ct.yaml b/.github/ct.yaml index 475449d..970c205 100644 --- a/.github/ct.yaml +++ b/.github/ct.yaml @@ -4,4 +4,6 @@ # - incubator=https://charts.helm.sh/incubator target-branch: main helm-extra-args: --debug +chart-repos: +- metrics-server=https://kubernetes-sigs.github.io/metrics-server debug: true diff --git a/charts/ocean-kubernetes-controller/Chart.yaml b/charts/ocean-kubernetes-controller/Chart.yaml index c030706..e574c9c 100644 --- a/charts/ocean-kubernetes-controller/Chart.yaml +++ b/charts/ocean-kubernetes-controller/Chart.yaml @@ -4,6 +4,7 @@ description: A Helm chart for Ocean Controller type: application version: 0.1.0 appVersion: 2.0.17 +kubeVersion: ">=1.20.0-0" maintainers: - name: spotinst email: ng-spot-info@netapp.com @@ -16,4 +17,6 @@ dependencies: - name: metrics-server version: 3.11.0 repository: https://kubernetes-sigs.github.io/metrics-server - condition: metrics-server.enabled \ No newline at end of file + condition: metrics-server.enabled +annotations: + artifacthub.io/prerelease: "true" diff --git a/charts/ocean-kubernetes-controller/templates/clusterrole.yaml b/charts/ocean-kubernetes-controller/templates/clusterrole.yaml index 3cfb62f..d571da3 100644 --- a/charts/ocean-kubernetes-controller/templates/clusterrole.yaml +++ b/charts/ocean-kubernetes-controller/templates/clusterrole.yaml @@ -85,9 +85,6 @@ rules: - apiGroups: ["apps"] resources: ["deployments", "daemonsets"] verbs: ["get", "list", "patch", "update", "create", "delete"] -- apiGroups: ["extensions"] - resources: ["daemonsets"] - verbs: ["get", "list", "patch", "update", "create", "delete"] - apiGroups: [""] resources: ["pods"] verbs: ["get", "list", "patch", "update", "create", "delete"] diff --git a/charts/ocean-kubernetes-controller/templates/deployment.yaml b/charts/ocean-kubernetes-controller/templates/deployment.yaml index 730572d..ca1dcf0 100644 --- a/charts/ocean-kubernetes-controller/templates/deployment.yaml +++ b/charts/ocean-kubernetes-controller/templates/deployment.yaml @@ -156,7 +156,6 @@ spec: {{- toYaml . | nindent 12 }} {{- end }} volumes: - # TODO: handle caBundleSecret volume {{- with .Values.extraVolumes }} {{- toYaml . | nindent 8 }} {{- end }} diff --git a/charts/ocean-kubernetes-controller/values.yaml b/charts/ocean-kubernetes-controller/values.yaml index 42ae504..9854e0d 100644 --- a/charts/ocean-kubernetes-controller/values.yaml +++ b/charts/ocean-kubernetes-controller/values.yaml @@ -85,8 +85,6 @@ securityContext: drop: - ALL -priorityClassName: system-cluster-critical - args: [] # - --test