From 3c61ae40ab5295801f5ca3eecf594ec76177225a Mon Sep 17 00:00:00 2001 From: Packet Please Date: Wed, 13 Sep 2023 01:16:07 +0200 Subject: [PATCH 01/17] buildbot: convert inventory to var encryption --- inventory/group_vars/buildbot | 415 ++++++++---------- .../buildbot.berlin.freifunk.net/main.yml | 28 +- 2 files changed, 199 insertions(+), 244 deletions(-) diff --git a/inventory/group_vars/buildbot b/inventory/group_vars/buildbot index 61bd46b..f10f9e2 100644 --- a/inventory/group_vars/buildbot +++ b/inventory/group_vars/buildbot @@ -1,230 +1,185 @@ -$ANSIBLE_VAULT;1.1;AES256 -32366533646666393632316439356631333361353938383464623439333434316135616533303265 -6261366566666464333139363761323231653833623466360a393434663162663264383234373930 -63333861363338636230373036636230636334343134376632376636633766636565393231643432 -3039643064343965370a383862303930383932383961616466353964303630336564316264396261 -62373738363531663336343136393033383664363763356564393232366232306663383135356136 -65626466663764346165376363303166303134623838346135653933396461646638393034633466 -34383366326364386265633265373566356131366130396461343633336638313031636364383031 -66613737623139313433633336356132316437626263663137396165613738326135643461383832 -61326565613839333865633963333266333235306337376362366231613634353966346436636265 -66363235343961386264393562393865366435613432383164623233333535633837333337373631 -34643437323361303266643662373463306163373864303533356633626162346536653836383165 -65633930653866323031633631386665653863386432356131666364356166613565646534356135 -62653733353034306364663635323162363065393230626564646430376635313635653430653632 -38626630616231393766656666313361333664623862653930333832643233336233393730663066 -33376663313663353135323661353939653739363864333033623063373532616666653534306462 -61666233353235646434353266343136326238333236323965663036316464653739343136306665 -32643037363136306238646264343736666536336334313835343133353437613030303362636665 -35393063393862353534333866393662613330373733336633333833626530323634313430643035 -64636438346236313932666164326463366238373334633531663735666431303137633065316461 -38636138643036313736373364633566366234306264656164383535363330336131366665613135 -32633066643566653764383833353962653831396230353332396138376464663065626431383135 -30656563333666306133653461653338663837663266306661333034623331623864616166383733 -66356333663961373138613032623335356266343531306233303362626337323832393461636531 -65323732653536373331386439376265653664313330396639623937626634343438333661326635 -64633636313733646332656161383161646561346134353232613336623939356339613964306438 -65653461363633323732303137633062616130353561646266623061623266363733316230636664 -31366234306538386234323034656261633439323034343738616331623866343739386161633639 -32306230383666646133356563656635333865323265636430386364326130623437363763303133 -62353833653434633166383563386337373036616264346664366137393732656565353562336434 -36663532376438306363316265396238363261616434333237636365663864646230393933356530 -38666635316166653065346462323839363765343735366666623530663836616430633565666439 -31613664613538336337343339653534373365653466316265306237323537363733323536356331 -63646137353337616433313339323339633761626664353736353836643335633438303365386238 -36306662646161633738316636613637303638643563313431663031663837646462343333366236 -38653830393935636139326165306537643030623565326638383436353238313238643164663239 -66333261623631633665383661376363313463646266646430353333333132303134306337616564 -31643361663437626561336136316365333339663839666562663537643436366531383362663064 -31313238393066323033636535396234663134633735363764663064376431623330356561633239 -37393335626362626539346466333037313364343732396334393238646339346335393663303464 -30386335316262396433396538663064373231323830316433383934393438353632373366323865 -61613831336432313035633438383065373537656636383232333565353165643838663637333262 -36613861343933366330313930616334326230333137626237623165376637626134363338303330 -32323236646664306230613437343637346634303332633466326532396433643938626436306166 -31663836366663373135613930663333653837336663396165383537386363666439613334653534 -36323163633435386130373235373139663163396165363534313031633963386334343837623662 -63623162303931313861636538393538383165613332363831303636306337646136643962366633 -35666164343533346364333134363765336261393961356564396666346663633434363134346635 -39373235363061343735306564373031623132656230383062323033613861303563623161383833 -66643030343937326362316335303866623764303033633433356566396461323964383731343363 -65323964333835633238623637323066613064313564643566613436343866383763363365653566 -61363562336139326564393231373463313936643235366661626439336362306137323134663932 -39613663353861323166303833663664646339656336323435373132373164623765666463336530 -61373266643262666331313462373739303830613962323364626662366466396339633764643732 -37646437393766303735633232353262343134633637346439623239376230393562353562303239 -64633062313635333965383363643466303165626431613361623266316661636234663864653938 -32616538303434653136353237363234646366383237346433363663346332356463396239343532 -35393462343434356638633833646566393131363261356438316534336233363036613965373132 -66356134666637376162613837383432383366613632366530353161303166623463376430313737 -36336362363738303362626634633935653866303931343361393933656331343831616466646361 -61363164636164643932666163303762633066303862326338333635353864343235366330383332 -30306232623733336263633463326130613462613266393863643164653438623130636364373736 -30633161333663616531393563343235346261303131316535633035356261313338376462623563 -35636438333630373137396537303766346163333861623563613139363132613638643736363262 -64333533323961613431623836313838333035623630383261616662646132356632356262303262 -38356465393038613563383365313862373333643562316465333333343834643830336362343832 -31663733346635316537613139383164356235356331656666373232303064663438656134663433 -65383438376238376137646665356566313465643838343161333230356230646538653137336564 -66326430333939663632306663623235386532323035636639636636333639623631383661363130 -66663165316333376430343636393831613739363430336663663765636561356435393736363239 -34366364373131616461303261626439616665336538393938326231376532383534656136373037 -36323633333430353638353261653034643261303464306135313839386331353733323039333865 -64366439363964363037323562666235313165363530663731343536333639333066323337366461 -32373834306539343461613739383834383365396466306438653938333134396134613563353833 -37323236313830336639393434643839326363313139346434653737626662343636356364313161 -37613335646337306565396562663733663634633862306338313862363066346462643262616562 -61643038666663666235633439613833303235383561636563326562353239396235396135386639 -34343036363037636266386634383435356635386138326438626365613033386134663461383366 -32316333326536306337646166623439656134653434656232613532326432333130653062623838 -37326131343130653137333738326363316261303363623232303566663465383036383830643263 -36666438323932373130303932366166343238396232303132636139616165346331383332653861 -65343638636430633939396232363164306631306334653366663538623437663061306130396333 -39326466333761316236663431393435643034326563366134623937383231633533313938636636 -38333632666137313137623436373639653132616437383438626632656236316433626334663962 -61373630623964373238623430376664333537623863306239663331633062396339373736323833 -63616231333566636131393561383239343466616638643930346135313739663334333138396232 -31643364633936326636366461613266643034643031633234633735353939326334313337333636 -65376337653533336538346639376431336366663933373131613836326561343263353162356364 -65656666383762653339396561303964333662366266616466303830363063366166343932636630 -66376262383633623165396633353134393035303232336566626466386630336639663136343830 -30306431353035613330633732666230623034353733623738646331366165353831626235343065 -62653137303063333963346232666334343164333566356261373831653738646338623035633230 -30343630633463653334653232323130623437363136373465353039346265336365343965303934 -62373330343062623763396334356336656664373161666530303336313064373463663230323762 -61613738333262643438616539393362313438353366663331626234303162336164623865333964 -39633966373564646266666166383336616530653361396338616564343763393330636632616537 -62613233616130323564356364643337373330653662333833326163366333633561313566306537 -61383531363037333533666130306363373633373961653066333066646237343633373463336662 -64376266306136616462636631313365386133363632323336366137666239653634366564626337 -37303139356464336561353836653831343663343762353534366664633037646235663737393231 -34363638303263373536656231376133653165376135626366383839663163613363386436623737 -30663261313237663134653934306231636436363138656336373931316232636665616135636433 -62343532333039333231336530653130616564393530373632353339366630386331326638663565 -35616635326239383962393663633163613663383930323837373230333437373261616131303134 -35363038343963643131303262653331313239616434353063343766336336646261663430323539 -32353034336261623838333933646437623932623931356630333232323162363262613262323961 -38393232356430333135366461306331396337363861313962326236306530386230336633346237 -39316633383161303232393033633332363566636562363232666566396234393561316336613364 -37616139666264656539626430613532323361323933333563663064626637306631366561393232 -30313131623438343763386436316433646431306233613731306433613166303063653035353463 -63643430306262303838373566653131613239646362333166663238613563646635323139336234 -32393461326230633165363631663165383730353239613132343936376165643434656439393330 -30343330303436313235383164656664336363396362363131636537633361653839333634613862 -62363861373433613330356334646334383936366165386632613333633132306565306165663632 -61346136306465323331643739376363623462386662663363616230613264363232636234343133 -34393930336139336261626338353663623638333261303730313936373466643465366135373962 -63303637646263613330656338346132336133643461323366356135306365393861353831343139 -30306435316366663063353636653432393531373361633262333366393339386661343964376638 -35616630623333373930363935323139323166366662613966373263633634336163333838663434 -33393330393464613364616364333265343463666433306635306562666135343932383133363432 -66663362336466646538343166353838333132613063363335623963386131386531616531336439 -32313335306230373933653339633534303965376264323739376536393436346132373163306639 -30393138376463333264333163656531646163666666326465343730326262343039376365666630 -62306234396532396166666131383366356663386563616336393438633861346433373265646466 -62303364356237313162316136393333316134333837326661386330336636633632666337313263 -37653237636364613234376631653039343463626465316230396632323961356235393636613034 -33323837373239636138653366316434373761373264373737633861663230313336336139326439 -35396163356439663332363432366339316162633263663266646538356434353632333336663363 -32663930356664616664666639333438303334306635653734643432303436633635663034373061 -63636539623531323638336631316365643635653433663263643438323134383232653837343461 -66363433353639303934303739643963346361326435623035373465663030633830636536336666 -63623532323733353866626661393664313037303964643264613239313463663936336436353562 -62396433396435633161633063643335336232616338363932666430663938366431346332383964 -30373033643437323161316632636535326364376564313633633034663861313134623134356262 -30613830643331336232303264323131333934666438393762303762653739623863316461373132 -32653831363434343964623839323863653935643565383736336563323137346639363364333430 -37626466316464353532363563323236336134363835383831353330353064386630313632613362 -38323530343335376164343234386636326333396666643063393130363935616263333461363133 -61316530303765373834646466373639363835356161356161616434666362366165623763386333 -63393039306661323632373562393339393834333038366536363064363030376334343165303131 -36356462636138633231306164386436323666316165386636396461636231336133643435386238 -32396637656236663139333935663962643162353734636538383635653032326238646334666437 -63653864363661666435393866663361323262353961323462353761373839316330393431343364 -30636131346636363461663561383031346138326263633139306433643234333765383037633863 -35626632303234373562323539643231353535656636336366386463333634323061643538613661 -39383666353237343334393339636535643130316630306263663437363138613235306631346634 -64373734663461396133646463393339306331326165313864616264653036626334353237363930 -33373265663532326138376437633435623630343131323534343336366266386362303462363162 -35393263386263386237393039376661333536333937646462616230386662303735393331313431 -34663133323266623331326134663436653765343937313363376233383035376134363034306330 -38333635623663663231633636303430326233333663343935656466303738383162386330396338 -30633933633361366630323765653538663834653737333466613434333862343161343661306533 -63326333636561366132363539613736303266373238343962393337643363663835663962613035 -34623365663336663037343134303730353561306334323664663332386433663136636430396637 -63376266326332623338633637613631333961613761656335616463346133326235363930306262 -62313562623064373734653262353036323735386361383632323637383833653439356261383336 -65356133343765386230373432386136636537343564333335393261653938376562366535373436 -62373736396632616338336364643463333430633063663633653064643661646337366330333231 -35623132643638326364373162666433613437323837356661353432653563346435393965356661 -30383339656532656130376231373666616239313730393338353033626266363865633061326332 -38666666626438383539653931646237656162653763646261316166386566633335616566386631 -66656232623837663861303262323338616130646531666661653263383335333565346134333230 -66373439646139333465393361303037333336363762383137383133363737383233313539363262 -37626461383334346637356464393063306632663461353138383537306231636138313765323136 -35346261323862353363383437666664333335376330363539626461373766393465343537363837 -61343563373336623163613234616166373762313330316139353361313263643134303730306632 -37636434303862393138373463333265643034333061373135333831386163646339326639366436 -61376337653564616134326564363261383637306137373164343739343661626138626331356534 -61323434333537313439653434326461623065633063383336633839396364643639316434646164 -39336635353566663533626462623934386632326535643566643038313833306139623062373361 -34653138313265653131356564663961613366643234623530366438313963363466646230636533 -66653966663465356462633062306663616231616230306365313833376662323663336432346532 -66356338313334363235363765646238343465393961633836373539636663363136383637376535 -33353837343236643437313564373164393639643937663535623764336266393363333166656332 -65663034623564646662333033633738306665613036613435353331386637633666643162343761 -64353461633637353035646561323835643965343137343765373335346461363834616262393839 -36353562643437386438656633336137336631316164396236323236393130613130663663386164 -64363033333561373461333532623937646634386235366131363737386435376564313034383135 -36666266633136396637373534636631393764366635366436623262636539343330303561376235 -39396530356237306665316630313962323164376133393438643761363461356165393739613330 -32383830383333366438633262346166663437643438626333396437653731356464326335306631 -33313030396634383331633134373435336463646339343834633066346166383634626266623866 -38383362326663353863356163343230306438383661343666373666396663633062343361323362 -64393632663032313834643039363538356563373434643131616635633366366339383132643032 -39663964343139316433303537626336363166613266633637333762383730316462663031396434 -38633266313937623961626439306264626466393630303134626166656164623564383938326431 -34346133643035366139353636643066323032666333306433346664616531313633303264666664 -61666562333835346362386134653762636364346464326333653062663866383166666361373730 -38656232316566396162323466353365316661633130623635636566353536623664393161323033 -66316438326266653434366237393762333135656130333765333133383336643330663937366136 -64346330383566646232626436326631353937393462653434633561353035303764623938343762 -36353934323839666435373264363864653638306239336465356638373234666333363832316136 -64356434326663306239313562363065313365663566626463373339346530663932356563316135 -30353131353130646636633563363039366362323334313339663434363065343031306531306164 -63653439666338376439343531323432636161663338636233373464663466386361346662636366 -66303064313538623765616462623339663061636530623133613765396131633834653864356233 -30646238376639303566333264336137656663396635323033626266623836333236656332323136 -64333935346431316466353264666434366132306539356433656234386335663532366263363062 -66623263376635393134626332643361666635356637663532656438623861366661383834313139 -38636630663135316438633366356261356539386664333032303230333131316437383461656133 -62313431383133323866666433356637363164333734323966346265366533643662353431363138 -35383562616431333739343936316466383131393061383563666139326331636632303538653438 -37643164313563663662343432616139636631653666333938373636326365356262346630383332 -66633862636462363239646130303063343134396664373662363863313566346363306636313034 -32643933633464623534396161323964643936313431363365373064626266646130626638396538 -63336434386231383635393635326662363931663539346365653563633031323661643761353434 -38306134386533646238333736653331363634313539356139346563323764326230313638316366 -32323036633961353263656334666566613636333662346435616331633633326265313661383538 -63313637353630613938343534303532313661306566633236346266323937373033623034316435 -61343863383464343561373136316663666462356630303866376163626432393433393232363865 -31616663396330356335636334393137363631363432643566396537393765653034343336303833 -64633238323230333366636434383339636137393835373039643334376430623734623235376532 -35303361373738383366346330343964303466366564393765303265306661646664643439623837 -64623936326234366466616130333939646230666462386432363235373738646232626631663338 -61393935393832396237613463666533373939343433613934616636313465323461383030613031 -34643532663430343631636330333234373738396132393237616534303639623739333161303732 -33616166376635393736656434383030656236383362626637613363663437353365353937323461 -61393262366233636561326234616334373562303961303765626433653334613364313062663735 -65363966393863303763626461366661613336383463646433636463326164363564633539326462 -34633134663966633339616638383037666363393466386266336137343434623466376663336261 -34333237343066633834306239333934363038653465393632376265353337356232663438393335 -38336362623465393335646665306263663163666337343139313262393237376130326662636166 -31383064393064316461623237636331316465336333616238643364633662373464346535643135 -33643630396264333739353339623534386462373931313536363963373831363337363065626661 -33316363616361626233343931643335653364633062373535383632336436646133313731353566 -38646539653061393535666636303636393363383230326533666539353362643130383635636138 -39623666613638613438623764366562323861356336343666656337326461383764323064363932 -32353535633566393561656535623761343266373932616232333034323365356434643833613666 -33346436656464393038616662316235353233303437353335666137323032383763623034343333 -3736 +--- +buildbot_version: "3.6.0" + +githubClientID: !vault | + $ANSIBLE_VAULT;1.1;AES256 + 32356638653865303261313531626638356232663461333063363531343266653063363565626534 + 3965386665336332666361393331666461636366303462630a613234396137323337383136343933 + 63623238393235393030353336653065663831636236623030323163376230623865616430333362 + 3338376132376231390a633737383362353166393766666366343238643235393463373732396265 + 33646461646235353963383938396134383162323430313163316432633435353431 +githubClientSecret: !vault | + $ANSIBLE_VAULT;1.1;AES256 + 61616434323231356234343365346638616134346137303434613363343735386361313831373230 + 3831343835663639636539336333343338663136336363300a323062306134326437623465623365 + 64653539326262333239333564336535623338633234663131373462353938633666653031623430 + 3936396466306437320a373830616430393136623565343331303535633936363134636665626237 + 32636631323864366264666639653939396433663832386431646336363863336432323339666662 + 3437393136366462383432323731666364393936326237336633 +webhook_secret: !vault | + $ANSIBLE_VAULT;1.1;AES256 + 64626335323662626530353262303735323135666433636239366362663332383064333834363932 + 3537333738343337613632646430383038333063326238300a353137656638373864336165303566 + 33353135646633623835643533333239363730393336323431373261636638353439326263313934 + 3730373565333632640a646236663864323431616536653036366661393035613562336536303733 + 66636531646132633439626165313664663063363839666631646638323138626634306464313231 + 65346333323732366162313833346334643764386138613961656462393736633334323563373066 + 623331343661316234316535376433643437 + +matrixHomeserver: https://matrix.org +matrixRoom: "!IBxpFGjLrqRPRFZXmu:matrix.org" # ffbot-dev +matrixAccessToken: !vault | + $ANSIBLE_VAULT;1.1;AES256 + 61366665626430383962376664393933653162346239616665633434626266633338656135646434 + 3962313064616431336365633738316366616362346464360a363731646134396136306337353539 + 34353462613835313737633964396464663339343864326133333532366333373333303931353835 + 3534333031666237340a363536613733346537623961643034613938306330313066623838313135 + 64386265356230343137363166633535613733306535363635303738303234366331376134386533 + 6238326434653865623735323330396437663465646166373330 + +buildbot_mariadb_user_password: !vault | + $ANSIBLE_VAULT;1.1;AES256 + 36366437343661323938376664653135383030656333636139323832623937663965623837306161 + 6237633733336464616330366435376361313537626237340a303631323066323135653264373834 + 32393862633230636230396662363966666234356162333838323837626564333862653662613337 + 3234653265383933330a633265643630396465303032306330353763636366386336626339646335 + 36626639323736316164356266666237366136616633333332306163396430366266613762636136 + 3761663136656365353533393965356364303132656638383665 + + +# variables to configure nginx-server on the buildbot-master machine + +nginx_remove_default_vhost: true + +nginx_extra_http_options: | + ssl_protocols TLSv1.2 TLSv1.3; + ssl_ciphers HIGH:!aNULL:!MD5; + +nginx_vhosts: + - listen: "80" + server_name: "buildbot.berlin.freifunk.net" + root: "/usr/local/src/www/htdocs" + index: "index.html index.htm" + extra_parameters: | + location ~ ^/.well-known { + root /usr/local/src/www/htdocs/; + } + location / { + include proxy_params; + proxy_pass http://localhost:8010; + } + + # Server sent event (sse) settings + location /sse { + proxy_buffering off; + proxy_pass http://localhost:8010; + } + + # Websocket settings + location /ws { + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_pass http://localhost:8010; + proxy_read_timeout 6000s; + } + filename: "buildbot.berlin.freifunk.net.conf" + + - listen: "443 ssl http2" + server_name: "buildbot.berlin.freifunk.net" + root: "/usr/local/src/www/htdocs" + index: "index.html index.htm" + extra_parameters: | + location ~ ^/.well-known { + root /usr/local/src/www/htdocs/; + } + location / { + include proxy_params; + proxy_pass http://localhost:8010; + } + + # Server sent event (sse) settings + location /sse { + proxy_buffering off; + proxy_pass http://localhost:8010; + } + + # Websocket settings + location /ws { + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_pass http://localhost:8010; + proxy_read_timeout 6000s; + } + ssl_certificate /etc/letsencrypt/live/buildbot/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/buildbot/privkey.pem; + filename: "buildbot.berlin.freifunk.net-ssl.conf" + + - listen: "80" + server_name: "firmware.berlin.freifunk.net" + root: "/usr/local/src/www/htdocs/buildbot" + index: "index index.html index.htm" + extra_parameters: | + autoindex on; + + location ~ ^/.well-known { + root /usr/local/src/www/htdocs/; + } + filename: "firmware.berlin.freifunk.net.conf" + + - listen: "443 ssl http2" + server_name: "firmware.berlin.freifunk.net" + root: "/usr/local/src/www/htdocs/buildbot" + index: "index index.html index.htm" + extra_parameters: | + autoindex on; + + location ~ ^/.well-known { + root /usr/local/src/www/htdocs/; + } + + ssl_certificate /etc/letsencrypt/live/buildbot/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/buildbot/privkey.pem; + filename: "firmware.berlin.freifunk.net-ssl.conf" + + - listen: "80" + server_name: "selector.berlin.freifunk.net" + root: "/usr/local/src/www/htdocs/firmware-selector/www" + index: "index.html" + extra_parameters: | + location ~ ^/misc { + root /usr/local/src/www/htdocs/firmware-selector/; + } + location ~ ^/.well-known { + root /usr/local/src/www/htdocs/; + } + filename: "selector.berlin.freifunk.net.conf" + + - listen: "443 ssl http2" + server_name: "selector.berlin.freifunk.net" + root: "/usr/local/src/www/htdocs/firmware-selector/www" + index: "index.html" + extra_parameters: | + location ~ ^/misc { + root /usr/local/src/www/htdocs/firmware-selector/; + } + location ~ ^/.well-known { + root /usr/local/src/www/htdocs/; + } + ssl_certificate /etc/letsencrypt/live/buildbot/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/buildbot/privkey.pem; + filename: "selector.berlin.freifunk.net-ssl.conf" + +letsencrypt_cert: + name: buildbot + domains: + - buildbot.berlin.freifunk.net + - firmware.berlin.freifunk.net + - selector.berlin.freifunk.net + challenge: http + http_auth: webroot + webroot_path: /usr/local/src/www/htdocs/ + services: + - nginx +letsencrypt_opts_extra: "--register-unsafely-without-email" diff --git a/inventory/host_vars/buildbot.berlin.freifunk.net/main.yml b/inventory/host_vars/buildbot.berlin.freifunk.net/main.yml index cbc9296..e559c11 100644 --- a/inventory/host_vars/buildbot.berlin.freifunk.net/main.yml +++ b/inventory/host_vars/buildbot.berlin.freifunk.net/main.yml @@ -1,14 +1,14 @@ -$ANSIBLE_VAULT;1.1;AES256 -35366637376531306361373131303633313938346633333539306465623739353864353162633939 -6337386335376532353931313735326462346239333662350a393438313434343964306336393864 -35633634363134343163623763326264366230643764653462373533363566366263376334383932 -3765333830303234360a303031313061396232623632356337313566323538373734323831663430 -66353639313561333437333562323765616261306662313135643663343565656438363135313466 -34383133643735373139386530613633383239653163636631346333343234383864333865626666 -62336663356563623638383561373536343538393532666461663231353532366364633065626135 -30633032313164386535313239376663376333646436306530386431353838323337363935396636 -65303737326531623463316430623931316535646331633938306361626530623861663931626465 -38666664346537393537393766383466633366643233643235306131333331353731633366623232 -37313538386465376366333562306566343031343461376339633165666635653337613763663462 -64633037656331353433623131346635366562313837616261313833363433303964333132343065 -62626231393839623565356565636635353434346237663563336239316639626637 +--- +bbworker_name: masterworker +bbworker_pwd: !vault | + $ANSIBLE_VAULT;1.1;AES256 + 35323564303562396566313735336535333937353937383333306338666565376463396432303330 + 6439333764633135363935353863383961343032363731330a656239353830316332613033663739 + 66353539653763333164616666343233656262393765386661653033306239376635333161303932 + 6136623236636335390a323763633261643338623435393866613339313332626466323639663738 + 37373863636162386430393636303433663238303965343237383032663562616563376333313132 + 36373639396238366534626330663137336233353531653331346239343337656238353036353131 + 316237353437373330343034626338396366 + +bbworker_contact: pktpls +bbworker_info: virtual worker, syncing builds etc. From a72e2508a504449a1ebff6058fdd987f774429d3 Mon Sep 17 00:00:00 2001 From: Packet Please Date: Wed, 13 Sep 2023 01:16:29 +0200 Subject: [PATCH 02/17] buildbot: packages build timeout not needed anymore --- roles/buildbot/files/packages.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/roles/buildbot/files/packages.py b/roles/buildbot/files/packages.py index 8a12bf0..412ed2a 100644 --- a/roles/buildbot/files/packages.py +++ b/roles/buildbot/files/packages.py @@ -169,14 +169,12 @@ def packagesArchFactory(f, wwwPrefix, wwwURL, alpineVersion): # * -i so we get output # * no -t because it messes with stdout/stderr # * --rm so we don't fill up the disk with old containers - # * --timeout so we don't accumulate hanging containers - # * requires Podman >= 3.2.0 # * --log-driver so we don't pump huge tarballs into the logging facility # * it'd also trigger a segfault in ~15% of concurrent runs: # https://github.com/containers/podman/issues/13779 # """\ -podman run -i --rm --timeout=1800 --log-driver=none docker.io/library/alpine:%(kw:alpineVersion)s sh -c '\ +podman run -i --rm --log-driver=none docker.io/library/alpine:%(kw:alpineVersion)s sh -c '\ ( \ apk add git bash wget xz gzip unzip grep diffutils findutils coreutils build-base gcc abuild binutils ncurses-dev gawk bzip2 perl python3 rsync argp-standalone musl-fts-dev musl-obstack-dev musl-libintl \ && git clone %(prop:repository)s /root/falter-packages \ From 48681d9d24c5cf4a68bf65fc21508f6d0974176e Mon Sep 17 00:00:00 2001 From: Packet Please Date: Wed, 13 Sep 2023 01:30:52 +0200 Subject: [PATCH 03/17] buildbot: update buildbot to 3.9.2 --- inventory/group_vars/buildbot | 3 ++- roles/buildbot_worker/defaults/main.yml | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/inventory/group_vars/buildbot b/inventory/group_vars/buildbot index f10f9e2..fb13632 100644 --- a/inventory/group_vars/buildbot +++ b/inventory/group_vars/buildbot @@ -1,5 +1,6 @@ --- -buildbot_version: "3.6.0" +# requires `buildbot upgrade-master` after every update +buildbot_version: "3.9.2" githubClientID: !vault | $ANSIBLE_VAULT;1.1;AES256 diff --git a/roles/buildbot_worker/defaults/main.yml b/roles/buildbot_worker/defaults/main.yml index 61e8034..cde2b4e 100644 --- a/roles/buildbot_worker/defaults/main.yml +++ b/roles/buildbot_worker/defaults/main.yml @@ -1,4 +1,6 @@ --- +buildbot_version: "3.9.2" + bbmaster_fqdn: buildbot.berlin.freifunk.net bbmaster_port: 9989 From 435f006ed2b22d61d4f40a40eb48d9bedfb9a8ee Mon Sep 17 00:00:00 2001 From: Packet Please Date: Tue, 19 Sep 2023 02:26:15 +0200 Subject: [PATCH 04/17] buildbot: remove old docker/podman dependencies --- inventory/hosts | 2 +- play.yml | 5 +---- requirements.yml | 6 ------ roles/buildbot_worker/tasks/main.yml | 31 ++-------------------------- 4 files changed, 4 insertions(+), 40 deletions(-) diff --git a/inventory/hosts b/inventory/hosts index 01cf35e..90510d1 100644 --- a/inventory/hosts +++ b/inventory/hosts @@ -24,4 +24,4 @@ f.tunnel.berlin.freifunk.net # vpn03h.berlin.freifunk.net t-löffel.de [other] -util.berlin.freifunk.net \ No newline at end of file +util.berlin.freifunk.net diff --git a/play.yml b/play.yml index 84617ad..e01eff9 100644 --- a/play.yml +++ b/play.yml @@ -26,12 +26,9 @@ - systemli.letsencrypt - name: Set up buildbot workers - # buildbot has docker already. Don't deploy podman for now. - hosts: buildbotworker, !buildbot.berlin.freifunk.net + hosts: buildbotworker become: true roles: - - alvistack.podman - - podman_fix - buildbot_worker - name: Set up config server diff --git a/requirements.yml b/requirements.yml index 2409329..4c02e12 100644 --- a/requirements.yml +++ b/requirements.yml @@ -6,12 +6,8 @@ roles: version: 1.0.8 - src: geerlingguy.nginx version: 3.1.0 - - src: geerlingguy.docker - version: 4.1.3 - src: systemli.letsencrypt version: 2.1.0 - - src: alvistack.podman - version: 6.3.1 collections: - name: ansible.posix version: 1.5.1 @@ -19,5 +15,3 @@ collections: version: 3.6.0 - name: community.general version: 6.5.0 - - name: community.docker - version: 3.4.3 diff --git a/roles/buildbot_worker/tasks/main.yml b/roles/buildbot_worker/tasks/main.yml index e30e0ab..063b358 100644 --- a/roles/buildbot_worker/tasks/main.yml +++ b/roles/buildbot_worker/tasks/main.yml @@ -1,39 +1,12 @@ --- + - name: Install dependencies for falter build system and openwrt apt: name: - - build-essential - - ccache - - curl - - ecj - - fastjar - - file - - g++ - - gawk - - gettext - git - - java-propose-classpath - - libelf-dev - - libncurses5-dev - - libncursesw5-dev - - libssl-dev - - python - - python2.7-dev - - python3 - - python3-dev - - python3-distutils - - python3-pip - - python3-setuptools - - rsync - rsync - - sqlite3 - - subversion - - swig - time - - unzip - - wget - - xsltproc - - zlib1g-dev + - podman state: present update_cache: true cache_valid_time: 3600 From 91ecd5c8a3623003c491b1de1f040bb2cb5e6e84 Mon Sep 17 00:00:00 2001 From: Packet Please Date: Tue, 19 Sep 2023 02:34:42 +0200 Subject: [PATCH 05/17] buildbot: install worker into a virtualenv --- roles/buildbot_worker/handlers/main.yml | 4 ++-- roles/buildbot_worker/tasks/main.yml | 11 ++++++++--- roles/buildbot_worker/templates/systemd-unit.j2 | 2 +- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/roles/buildbot_worker/handlers/main.yml b/roles/buildbot_worker/handlers/main.yml index c1c692e..6d25a9b 100644 --- a/roles/buildbot_worker/handlers/main.yml +++ b/roles/buildbot_worker/handlers/main.yml @@ -7,5 +7,5 @@ state: restarted - name: Initialise buildbot-worker - command: buildbot-worker create-worker --umask=0o22 {{ bbworker_path }} {{ bbmaster_fqdn }}:{{ bbmaster_port }} {{ bbworker_name }} {{ bbworker_pwd }} - become_user: bbworker + command: "{{ bbworker_path }}/env/bin/buildbot-worker create-worker --umask=0o22 {{ bbworker_path }} {{ bbmaster_fqdn }}:{{ bbmaster_port }} {{ bbworker_name }} {{ bbworker_pwd }}" + become_user: "{{ machine_user }}" diff --git a/roles/buildbot_worker/tasks/main.yml b/roles/buildbot_worker/tasks/main.yml index 063b358..a9eb05e 100644 --- a/roles/buildbot_worker/tasks/main.yml +++ b/roles/buildbot_worker/tasks/main.yml @@ -4,6 +4,7 @@ apt: name: - git + - virtualenv - rsync - time - podman @@ -17,10 +18,14 @@ - name: Install buildbot-worker pip: - name: - - buildbot-worker + name: buildbot-worker + version: "{{ buildbot_version }}" state: present - notify: Initialise buildbot-worker + virtualenv: "{{ bbworker_path }}env" + become_user: "{{ machine_user }}" + notify: + - Initialise buildbot-worker + - Restart buildbot-worker - name: Flush handlers meta: flush_handlers diff --git a/roles/buildbot_worker/templates/systemd-unit.j2 b/roles/buildbot_worker/templates/systemd-unit.j2 index 5096ffe..222fbe5 100644 --- a/roles/buildbot_worker/templates/systemd-unit.j2 +++ b/roles/buildbot_worker/templates/systemd-unit.j2 @@ -8,7 +8,7 @@ After=network.target User=bbworker Group=bbworker WorkingDirectory={{ bbworker_path }} -ExecStart=/usr/local/bin/buildbot-worker start --nodaemon +ExecStart={{ bbworker_path }}/env/bin/buildbot-worker start --nodaemon [Install] WantedBy=multi-user.target From f3d4b2ccfdff728cd9bebe767136a060ecbddf68 Mon Sep 17 00:00:00 2001 From: Packet Please Date: Tue, 19 Sep 2023 02:35:56 +0200 Subject: [PATCH 06/17] buildbot: more resilient on package upgrades --- roles/buildbot/handlers/main.yml | 6 +++++- roles/buildbot/tasks/main.yml | 11 +++++++---- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/roles/buildbot/handlers/main.yml b/roles/buildbot/handlers/main.yml index 818837e..0066f78 100644 --- a/roles/buildbot/handlers/main.yml +++ b/roles/buildbot/handlers/main.yml @@ -7,7 +7,11 @@ state: restarted - name: Initialize buildbot - command: /usr/local/src/buildbot/env/bin/buildbot create-master /usr/local/src/buildbot/masters/master + command: /usr/local/src/buildbot/env/bin/buildbot create-master -f /usr/local/src/buildbot/masters/master + become_user: buildbot + +- name: Upgrade buildbot database + command: /usr/local/src/buildbot/env/bin/buildbot upgrade-master /usr/local/src/buildbot/masters/master become_user: buildbot - name: Restart mariadb diff --git a/roles/buildbot/tasks/main.yml b/roles/buildbot/tasks/main.yml index da0d4ef..7472328 100644 --- a/roles/buildbot/tasks/main.yml +++ b/roles/buildbot/tasks/main.yml @@ -3,11 +3,12 @@ apt: name: - build-essential - - default-libmysqlclient-dev + - pkg-config - git - mariadb-server - - python3-dev + - default-libmysqlclient-dev - python3-mysqldb + - python3-dev - virtualenv state: present update_cache: true @@ -62,14 +63,16 @@ # - txrequests # the software stated, that treq would be faster by 2.8 times or so... - treq virtualenv: /usr/local/src/buildbot/env - notify: Initialise buildbot - name: Install buildbot in venv pip: name: buildbot[bundle] version: "{{ buildbot_version }}" virtualenv: /usr/local/src/buildbot/env - notify: Initialise buildbot + notify: + - Initialise buildbot + - Upgrade buildbot database + - Restart buildbot - name: Install mysql dependencies in venv pip: From 4f53d5c4f3a8242481a1aa522a3840380cca752a Mon Sep 17 00:00:00 2001 From: Packet Please Date: Wed, 20 Sep 2023 02:51:21 +0200 Subject: [PATCH 07/17] fmt: install black and isort formatters --- requirements.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/requirements.txt b/requirements.txt index 2ad1fba..63d2af9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,3 @@ ansible >= 2.13 +black >= 23.9 +isort >= 5.12 From ff3feba9e0c4136e3d535c8b4a6d0414f961b0cd Mon Sep 17 00:00:00 2001 From: Packet Please Date: Wed, 20 Sep 2023 02:51:29 +0200 Subject: [PATCH 08/17] fmt: run black and isort --- roles/buildbot/files/asyncbuild.py | 23 +- .../files/generate_autoupdate_json.py | 28 +- roles/buildbot/files/notify.py | 81 +-- roles/buildbot/files/packages.py | 341 ++++++++----- roles/buildbot/files/targets.py | 469 +++++++++++------- roles/ff_monitor/files/create_node_list.py | 15 +- 6 files changed, 583 insertions(+), 374 deletions(-) diff --git a/roles/buildbot/files/asyncbuild.py b/roles/buildbot/files/asyncbuild.py index bc1f4a9..08d4436 100644 --- a/roles/buildbot/files/asyncbuild.py +++ b/roles/buildbot/files/asyncbuild.py @@ -1,12 +1,13 @@ # -*- python -*- # ex: set filetype=python: -from buildbot.plugins import util, steps +from typing import List + +from buildbot.plugins import steps, util from buildbot.process import build, buildstep, factory, logobserver from twisted.internet import defer from twisted.python import log -from typing import List # AsyncBuildGenerator dynamically generates build steps from command output. # @@ -19,14 +20,14 @@ def __init__(self, stepFunc, **kwargs): kwargs = self.setupShellMixin(kwargs) super().__init__(**kwargs) self.observer = logobserver.BufferLogObserver() - self.addLogObserver('stdio', self.observer) + self.addLogObserver("stdio", self.observer) self.stepFunc = stepFunc def getLines(self, stdout): archs = [] - for line in stdout.split('\n'): + for line in stdout.split("\n"): arch = str(line.strip()) - if arch and not arch.startswith('#'): + if arch and not arch.startswith("#"): archs.append(arch) return archs @@ -36,11 +37,12 @@ def run(self): yield self.runCommand(cmd) result = cmd.results() if result == util.SUCCESS: - self.build.addStepsAfterCurrentStep([ - self.stepFunc(a) for a in self.getLines(self.observer.getStdout()) - ]) + self.build.addStepsAfterCurrentStep( + [self.stepFunc(a) for a in self.getLines(self.observer.getStdout())] + ) return result + # AsyncTrigger is a Trigger step which executes in parallel with other AsyncTriggers. # It's a useful middleground between waitForFinish=False and waitForFinish=True. # @@ -60,8 +62,8 @@ def _createStep(self): self.name = yield self.build.render(self.name) self.build.setUniqueStepName(self) self.stepid, self.number, self.name = yield self.master.data.updates.addStep( - buildid=self.build.buildid, - name=self.name) + buildid=self.build.buildid, name=self.name + ) @defer.inlineCallbacks def addStep(self): @@ -91,6 +93,7 @@ def run(self): return results + # AsyncBuild is a Build which can execute AsyncTrigger steps in parallel. # It's a useful middleground between waitForFinish=False and waitForFinish=True. # diff --git a/roles/buildbot/files/generate_autoupdate_json.py b/roles/buildbot/files/generate_autoupdate_json.py index e647032..c9f5578 100644 --- a/roles/buildbot/files/generate_autoupdate_json.py +++ b/roles/buildbot/files/generate_autoupdate_json.py @@ -6,23 +6,33 @@ import json import os - parser = argparse.ArgumentParser() -parser.add_argument('-v', dest='version', type=str, required=True, - help='falter-version that the autoupdate-file is for.') -parser.add_argument('-p', dest='dir', type=str, required=True, - help='version directory, in which there are tunneldigger- and notunnel- dirs. autoupdate.json will be placed in that directory too.') +parser.add_argument( + "-v", + dest="version", + type=str, + required=True, + help="falter-version that the autoupdate-file is for.", +) +parser.add_argument( + "-p", + dest="dir", + type=str, + required=True, + help="version directory, in which there are tunneldigger- and notunnel- dirs. autoupdate.json will be placed in that directory too.", +) args = parser.parse_args() # build autoupdate.json autoupdate_json = {} autoupdate_json[ - "image_url"] = "https://firmware.berlin.freifunk.net/stable/{falter-version}/{flavour}/{target}" + "image_url" +] = "https://firmware.berlin.freifunk.net/stable/{falter-version}/{flavour}/{target}" autoupdate_json["target"] = {} autoupdate_json["falter-version"] = args.version # get paths of all json-files -file_list = os.popen('find ' + args.dir + ' -name "*.json"').read().split() +file_list = os.popen("find " + args.dir + ' -name "*.json"').read().split() # aggregate the content of all files for fpath in file_list: @@ -36,12 +46,12 @@ # print(profiles) # get flavour and omit backbone-images - flavour = fpath.split('/')[-4] + flavour = fpath.split("/")[-4] if flavour == "backbone": continue # get target and create dict, if not already created - target = '/'.join(fpath.split('/')[-3:-1]) + target = "/".join(fpath.split("/")[-3:-1]) if autoupdate_json.get("target").get(target) == None: autoupdate_json["target"][target] = {} diff --git a/roles/buildbot/files/notify.py b/roles/buildbot/files/notify.py index 9c1c4d6..3933004 100644 --- a/roles/buildbot/files/notify.py +++ b/roles/buildbot/files/notify.py @@ -1,37 +1,38 @@ # -*- python -*- # ex: set filetype=python: -from buildbot.plugins import util, steps +import pprint + +from buildbot.plugins import steps, util +from buildbot.process.results import statusToString from buildbot.reporters.base import ReporterBase from buildbot.reporters.generators.build import BuildStartEndStatusGenerator from buildbot.reporters.generators.buildrequest import BuildRequestGenerator from buildbot.reporters.message import MessageFormatterRenderable from buildbot.util import httpclientservice -from buildbot.process.results import statusToString - from twisted.internet import defer from twisted.python import log -import pprint class MatrixNotifier(ReporterBase): - def __init__(self, homeserver=None, accessToken=None, room=None, **kwargs): self.homeserver = homeserver self.accessToken = accessToken self.room = room super().__init__(**kwargs) - def checkConfig(self, context=None, debug=None, verify=None, generators=None, - **kwargs): + def checkConfig( + self, context=None, debug=None, verify=None, generators=None, **kwargs + ): if generators is None: generators = self._create_default_generators() super().checkConfig(generators=generators, **kwargs) @defer.inlineCallbacks - def reconfigService(self, context=None, debug=None, - verify=None, generators=None, **kwargs): + def reconfigService( + self, context=None, debug=None, verify=None, generators=None, **kwargs + ): self.debug = debug self.verify = verify self.context = self.setup_context(context) @@ -41,50 +42,62 @@ def reconfigService(self, context=None, debug=None, yield super().reconfigService(generators=generators, **kwargs) def setup_context(self, context): - return context or util.Interpolate('buildbot/%(prop:buildername)s') + return context or util.Interpolate("buildbot/%(prop:buildername)s") # This is needed but I have no clue what it does. def _create_default_generators(self): - start_formatter = MessageFormatterRenderable('Build started.') - end_formatter = MessageFormatterRenderable('Build done.') - pending_formatter = MessageFormatterRenderable('Build pending.') + start_formatter = MessageFormatterRenderable("Build started.") + end_formatter = MessageFormatterRenderable("Build done.") + pending_formatter = MessageFormatterRenderable("Build pending.") return [ BuildRequestGenerator(formatter=pending_formatter), - BuildStartEndStatusGenerator(start_formatter=start_formatter, - end_formatter=end_formatter) + BuildStartEndStatusGenerator( + start_formatter=start_formatter, end_formatter=end_formatter + ), ] @defer.inlineCallbacks def sendMessage(self, reports): - b = reports[0]['builds'][0] - builder = b['builder']['name'] - - if 'number' not in b or b['complete'] != True or (builder != 'builds/targets' and builder != 'builds/packages'): + b = reports[0]["builds"][0] + builder = b["builder"]["name"] + + if ( + "number" not in b + or b["complete"] != True + or (builder != "builds/targets" and builder != "builds/packages") + ): return # pp = pprint.PrettyPrinter(indent=4) # log.msg(f'MatrixNotifier.sendMessage - {pp.pformat(reports)}') - version = b['properties']['branch'][0] - if builder == 'builds/targets': - version = b['properties']['falterVersion'][0] + version = b["properties"]["branch"][0] + if builder == "builds/targets": + version = b["properties"]["falterVersion"][0] - result = statusToString(b['results']) - color = '#ff0000' - if b['results'] == 0: - color = '#008000' + result = statusToString(b["results"]) + color = "#ff0000" + if b["results"] == 0: + color = "#008000" - url = b['url'] + url = b["url"] - msg = f'{builder} @ {version} · {result} · {url}' - htmlmsg = f'{builder} @ {version} · {result} · {url}' + msg = f"{builder} @ {version} · {result} · {url}" + htmlmsg = ( + f'{builder} @ {version} · {result} · {url}' + ) http = yield httpclientservice.HTTPClientService.getService( - self.master, self.homeserver) + self.master, self.homeserver + ) res = http.post( - f'/_matrix/client/r0/rooms/{self.room}/send/m.room.message?access_token={self.accessToken}', - json={'msgtype':'m.notice', 'body':msg, - 'format':'org.matrix.custom.html', 'formatted_body':htmlmsg}) - + f"/_matrix/client/r0/rooms/{self.room}/send/m.room.message?access_token={self.accessToken}", + json={ + "msgtype": "m.notice", + "body": msg, + "format": "org.matrix.custom.html", + "formatted_body": htmlmsg, + }, + ) diff --git a/roles/buildbot/files/packages.py b/roles/buildbot/files/packages.py index 412ed2a..0cba7b1 100644 --- a/roles/buildbot/files/packages.py +++ b/roles/buildbot/files/packages.py @@ -1,80 +1,107 @@ # -*- python -*- # ex: set filetype=python: -from buildbot.plugins import * import re from asyncbuild import * +from buildbot.plugins import * + def packagesConfig(c, config): + c["schedulers"].append( + schedulers.Triggerable(name="dummy/packages", builderNames=["dummy/packages"]) + ) + + c["schedulers"].append( + schedulers.ForceScheduler( + name="force-packages", + buttonName="Build Package Feed", + builderNames=["builds/packages"], + codebases=[ + util.CodebaseParameter( + "", + label="Build falter package feeds using falter-packages.git", + branch=util.ChoiceStringParameter( + name="branch", + label="git branch", + choices=config["packages_branches"], + default="master", + strict=True, + ), + revision=util.FixedParameter(name="revision", default=""), + repository=util.FixedParameter( + name="repository", default=config["packages_repo"] + ), + project=util.FixedParameter(name="project", default=""), + ) + ], + ) + ) + + c["builders"].append( + util.BuilderConfig( + name="builds/packages", + workernames=["masterworker"], + factory=packagesFactory(util.BuildFactory(), config["publishDir"]), + collapseRequests=False, + ) + ) + + c["builders"].append( + util.BuilderConfig( + name="dummy/packages", + workernames=config["workerNames"], + factory=packagesArchFactory( + util.BuildFactory(), + config["publishDir"], + config["publishURL"], + config["alpineVersion"], + ), + collapseRequests=False, + ) + ) + + return c - c['schedulers'].append(schedulers.Triggerable( - name="dummy/packages", - builderNames=["dummy/packages"])) - - c['schedulers'].append(schedulers.ForceScheduler( - name="force-packages", - buttonName="Build Package Feed", - builderNames=["builds/packages"], - codebases=[ - util.CodebaseParameter( - "", - label="Build falter package feeds using falter-packages.git", - branch=util.ChoiceStringParameter( - name="branch", - label="git branch", - choices=config['packages_branches'], - default="master", - strict=True), - revision=util.FixedParameter(name="revision", default=""), - repository=util.FixedParameter(name="repository", default=config['packages_repo']), - project=util.FixedParameter(name="project", default=""))])) - - c['builders'].append(util.BuilderConfig( - name="builds/packages", - workernames=["masterworker"], - factory=packagesFactory(util.BuildFactory(), config['publishDir']), - collapseRequests=False)) - - c['builders'].append(util.BuilderConfig( - name="dummy/packages", - workernames=config['workerNames'], - factory=packagesArchFactory(util.BuildFactory(), config['publishDir'], config['publishURL'], config['alpineVersion']), - collapseRequests=False)) - - return c # Passed by packagesFactory to AsyncBuildGenerator to be called for each arch. def archTriggerStep(arch): - return AsyncTrigger( - # Step name limit is 50 chars, longest is currently 33 chars: - # "trigger arm_cortex-a15_neon-vfpv4" - # See https://github.com/buildbot/buildbot/issues/3413 - name=util.Interpolate("trigger %(kw:arch)s", arch=arch), - waitForFinish=True, - warnOnFailure=True, - schedulerNames=["dummy/packages"], - copy_properties=['repository', 'branch', 'revision', 'got_revision'], - set_properties={ - 'arch': arch, - 'origbuildnumber': util.Interpolate("%(prop:buildnumber)s"), - # Builder name limit is 70 characters, longest is currently 48 chars: - # packages/openwrt-22.03/arm_cortex-a15_neon-vfpv4 - # See https://github.com/buildbot/buildbot/pull/3957 - 'virtual_builder_name': util.Interpolate("packages/%(prop:branch)s/%(kw:arch)s", arch=arch), - 'virtual_builder_tags': ["packages", util.Interpolate("%(prop:branch)s")], - }) + return AsyncTrigger( + # Step name limit is 50 chars, longest is currently 33 chars: + # "trigger arm_cortex-a15_neon-vfpv4" + # See https://github.com/buildbot/buildbot/issues/3413 + name=util.Interpolate("trigger %(kw:arch)s", arch=arch), + waitForFinish=True, + warnOnFailure=True, + schedulerNames=["dummy/packages"], + copy_properties=["repository", "branch", "revision", "got_revision"], + set_properties={ + "arch": arch, + "origbuildnumber": util.Interpolate("%(prop:buildnumber)s"), + # Builder name limit is 70 characters, longest is currently 48 chars: + # packages/openwrt-22.03/arm_cortex-a15_neon-vfpv4 + # See https://github.com/buildbot/buildbot/pull/3957 + "virtual_builder_name": util.Interpolate( + "packages/%(prop:branch)s/%(kw:arch)s", arch=arch + ), + "virtual_builder_tags": ["packages", util.Interpolate("%(prop:branch)s")], + }, + ) + # Only needed for publishing, see pubdir further below. # Apart from that only used for targets builds. @util.renderer def branchToFalterBranch(props): - o2f = { 'master':'snapshot', - 'openwrt-23.05':'1.4.0-snapshot', - 'openwrt-22.03':'1.3.0-snapshot', - 'openwrt-21.02':'1.2.3-snapshot', - 'testbuildbot':'testbuildbot' } - return o2f.get(props['branch']) + o2f = { + "master": "snapshot", + "openwrt-23.05": "1.4.0-snapshot", + "openwrt-22.03": "1.3.0-snapshot", + "openwrt-21.02": "1.2.3-snapshot", + "testbuildbot": "testbuildbot", + } + return o2f.get(props["branch"]) + # Fans out to one builder per arch and blocks for the results. def packagesFactory(f, wwwPrefix): @@ -83,7 +110,9 @@ def packagesFactory(f, wwwPrefix): steps.SetProperty( name=util.Interpolate("falterBranch = %(kw:fv)s", fv=branchToFalterBranch), property="falterBranch", - value=branchToFalterBranch)) + value=branchToFalterBranch, + ) + ) f.addStep( steps.Git( name="git clone", @@ -91,89 +120,113 @@ def packagesFactory(f, wwwPrefix): repourl=util.Interpolate("%(prop:repository)s"), # if using incremental, we get strange behaviors, when configuring # another repo, that is back the other repo. Doing full checkouts is cleaner - method='clobber', - mode='full', - submodules=True)) + method="clobber", + mode="full", + submodules=True, + ) + ) f.addStep( - AsyncBuildGenerator(archTriggerStep, + AsyncBuildGenerator( + archTriggerStep, name="generate builds", haltOnFailure=True, - command=["sh", "-c", util.Interpolate( - # Read our list of package architectures to build: - # - # 1. Print the architectures-to-targets mapping - # 2. Remove comments and empty lines - # 3. Take only the first column (delimited by a space) - # - # TODO: doesn't fail if targets-*.txt doesn't exist - """\ + command=[ + "sh", + "-c", + util.Interpolate( + # Read our list of package architectures to build: + # + # 1. Print the architectures-to-targets mapping + # 2. Remove comments and empty lines + # 3. Take only the first column (delimited by a space) + # + # TODO: doesn't fail if targets-*.txt doesn't exist + """\ cat build/targets-%(prop:branch)s.txt \ | grep -v '#' | grep . \ | cut -d' ' -f1 \ -""")])) +""" + ), + ], + ) + ) wwwdir = util.Interpolate( - "%(kw:prefix)s/builds/packages/%(prop:buildnumber)s", prefix=wwwPrefix) + "%(kw:prefix)s/builds/packages/%(prop:buildnumber)s", prefix=wwwPrefix + ) pubdir = util.Interpolate( - "%(kw:prefix)s/feed/%(prop:falterBranch)s/packages", prefix=wwwPrefix) + "%(kw:prefix)s/feed/%(prop:falterBranch)s/packages", prefix=wwwPrefix + ) f.addStep( steps.MasterShellCommand( name="publish", haltOnFailure=True, - command=["sh", "-c", util.Interpolate( - # Publish build in a way that minimizes downtime. - # - # We call the currently published artifacts "current", - # which is e.g. /unstable/1.3.0-snapshot or /unstable/snapshot. - # We also use two temporary directories called "new" and "prev". - # - # 1. Remove "new" and "prev" leftovers from previous builds - # 2. Move build artifacts into "new" - # 3. Rename "current" published stuff to "prev" - # 4. Publish "new" by renaming it to "current" - # 5. Remove "prev" - # - # This is slightly different from targets builds, - # where we only move instead of copy+move. - # That means /builds/packages/%d is available after publishing, - # while it's empty for targets builds. - # - # Packages downloads are only unavailable after step 3 and - # before step 4 has completed. - # - # Just symlinking from /builds/packages/%d is not a good option, - # since we want to be able to just delete that at any time, - # without worrying about symlinks pointing to deleted stuff. - """\ + command=[ + "sh", + "-c", + util.Interpolate( + # Publish build in a way that minimizes downtime. + # + # We call the currently published artifacts "current", + # which is e.g. /unstable/1.3.0-snapshot or /unstable/snapshot. + # We also use two temporary directories called "new" and "prev". + # + # 1. Remove "new" and "prev" leftovers from previous builds + # 2. Move build artifacts into "new" + # 3. Rename "current" published stuff to "prev" + # 4. Publish "new" by renaming it to "current" + # 5. Remove "prev" + # + # This is slightly different from targets builds, + # where we only move instead of copy+move. + # That means /builds/packages/%d is available after publishing, + # while it's empty for targets builds. + # + # Packages downloads are only unavailable after step 3 and + # before step 4 has completed. + # + # Just symlinking from /builds/packages/%d is not a good option, + # since we want to be able to just delete that at any time, + # without worrying about symlinks pointing to deleted stuff. + """\ mkdir -p %(kw:p)s %(kw:p)s.new \ && cp -a %(kw:w)s/* %(kw:p)s.new/ \ && mv %(kw:p)s %(kw:p)s.prev \ && mv %(kw:p)s.new %(kw:p)s \ && rm -rf %(kw:p)s.prev \ """, - w=wwwdir, p=pubdir)])) + w=wwwdir, + p=pubdir, + ), + ], + ) + ) return f + # Runs build.sh with prop:arch and prop:branch, and uploads the result to master. def packagesArchFactory(f, wwwPrefix, wwwURL, alpineVersion): f.addStep( steps.ShellCommand( name="build", haltOnFailure=True, - interruptSignal='TERM', # podman can't proxy the KILL signal - command=["sh", "-c", util.Interpolate( - # Container that prints a tarball to stdout, and build output to stderr. - # - # Parameters: - # * -i so we get output - # * no -t because it messes with stdout/stderr - # * --rm so we don't fill up the disk with old containers - # * --log-driver so we don't pump huge tarballs into the logging facility - # * it'd also trigger a segfault in ~15% of concurrent runs: - # https://github.com/containers/podman/issues/13779 - # - """\ + interruptSignal="TERM", # podman can't proxy the KILL signal + command=[ + "sh", + "-c", + util.Interpolate( + # Container that prints a tarball to stdout, and build output to stderr. + # + # Parameters: + # * -i so we get output + # * no -t because it messes with stdout/stderr + # * --rm so we don't fill up the disk with old containers + # * --log-driver so we don't pump huge tarballs into the logging facility + # * it'd also trigger a segfault in ~15% of concurrent runs: + # https://github.com/containers/podman/issues/13779 + # + """\ podman run -i --rm --log-driver=none docker.io/library/alpine:%(kw:alpineVersion)s sh -c '\ ( \ apk add git bash wget xz gzip unzip grep diffutils findutils coreutils build-base gcc abuild binutils ncurses-dev gawk bzip2 perl python3 rsync argp-standalone musl-fts-dev musl-obstack-dev musl-libintl \ @@ -186,11 +239,18 @@ def packagesArchFactory(f, wwwPrefix, wwwURL, alpineVersion): ) >&2 \ && cd /root/falter-packages/out/ \ && tar -c *' > out.tar \ -""", alpineVersion=alpineVersion)])) +""", + alpineVersion=alpineVersion, + ), + ], + ) + ) tarfile = util.Interpolate("packages-%(prop:origbuildnumber)s-%(prop:arch)s.tar") wwwpath = util.Interpolate("builds/packages/%(prop:origbuildnumber)s/%(prop:arch)s") - wwwdir = util.Interpolate("%(kw:prefix)s/%(kw:wwwpath)s", prefix=wwwPrefix, wwwpath=wwwpath) + wwwdir = util.Interpolate( + "%(kw:prefix)s/%(kw:wwwpath)s", prefix=wwwPrefix, wwwpath=wwwpath + ) wwwurl = util.Interpolate("%(kw:url)s/%(kw:wwwpath)s", url=wwwURL, wwwpath=wwwpath) f.addStep( steps.FileUpload( @@ -199,34 +259,57 @@ def packagesArchFactory(f, wwwPrefix, wwwURL, alpineVersion): workersrc="out.tar", masterdest=tarfile, url=wwwurl, - urlText=wwwurl)) + urlText=wwwurl, + ) + ) f.addStep( steps.MasterShellCommand( name="extract", haltOnFailure=True, - command=["sh", "-c", util.Interpolate( - "mkdir -vp %(kw:wwwdir)s && tar -v -C %(kw:wwwdir)s -xf %(kw:tarfile)s", - tarfile=tarfile, wwwdir=wwwdir)])) + command=[ + "sh", + "-c", + util.Interpolate( + "mkdir -vp %(kw:wwwdir)s && tar -v -C %(kw:wwwdir)s -xf %(kw:tarfile)s", + tarfile=tarfile, + wwwdir=wwwdir, + ), + ], + ) + ) f.addStep( steps.MasterShellCommand( name="sign", haltOnFailure=True, - command=["sh", "-c", util.Interpolate( - "signify-openbsd -S -m %(kw:wwwdir)s/falter/Packages -s packagefeed_master.sec", - wwwdir=wwwdir)])) + command=[ + "sh", + "-c", + util.Interpolate( + "signify-openbsd -S -m %(kw:wwwdir)s/falter/Packages -s packagefeed_master.sec", + wwwdir=wwwdir, + ), + ], + ) + ) f.addStep( steps.MasterShellCommand( name="cleanup master", alwaysRun=True, warnOnFailure=False, - command=["sh", "-c", util.Interpolate( - "rm -vf %(kw:tarfile)s", - tarfile=tarfile)])) + command=[ + "sh", + "-c", + util.Interpolate("rm -vf %(kw:tarfile)s", tarfile=tarfile), + ], + ) + ) f.addStep( steps.ShellCommand( name="cleanup worker", alwaysRun=True, warnOnFailure=False, - command=["sh", "-c", "rm -vf out.tar"])) + command=["sh", "-c", "rm -vf out.tar"], + ) + ) return f diff --git a/roles/buildbot/files/targets.py b/roles/buildbot/files/targets.py index cba9f13..ac7c8ef 100644 --- a/roles/buildbot/files/targets.py +++ b/roles/buildbot/files/targets.py @@ -1,127 +1,176 @@ # -*- python -*- # ex: set filetype=python: +from asyncbuild import * from buildbot.plugins import * -from asyncbuild import * def targetsConfig(c, config): + c["schedulers"].append( + schedulers.Triggerable(name="dummy/targets", builderNames=["dummy/targets"]) + ) - c['schedulers'].append(schedulers.Triggerable( - name="dummy/targets", - builderNames=["dummy/targets"])) + c["schedulers"].append( + schedulers.ForceScheduler( + name="force-targets", + label="Snapshot", + buttonName="Build Snapshot", + builderNames=["builds/targets"], + codebases=[ + util.CodebaseParameter( + "", + label="Build falter snapshot images using falter-builter.git", + branch=util.ChoiceStringParameter( + name="branch", + label="git branch", + choices=config["builter_branches"], + default=config["builter_branches"][0], + strict=True, + ), + revision=util.FixedParameter(name="revision", default=""), + repository=util.FixedParameter( + name="repository", default=config["builter_repo"] + ), + project=util.FixedParameter(name="project", default=""), + ) + ], + properties=[ + util.ChoiceStringParameter( + name="falterBranch", + label="falter release branch", + choices=config["builter_releaseBranches"], + default=config["builter_releaseBranches"][0], + strict=True, + ) + ], + reason=util.FixedParameter(name="reason", default="manual", hide=True), + ) + ) - c['schedulers'].append(schedulers.ForceScheduler( - name="force-targets", - label="Snapshot", - buttonName="Build Snapshot", - builderNames=["builds/targets"], - codebases=[ - util.CodebaseParameter( - "", - label="Build falter snapshot images using falter-builter.git", - branch=util.ChoiceStringParameter( - name="branch", - label="git branch", - choices=config['builter_branches'], - default=config['builter_branches'][0], - strict=True), - revision=util.FixedParameter(name="revision", default=""), - repository=util.FixedParameter(name="repository", default=config['builter_repo']), - project=util.FixedParameter(name="project", default=""))], - properties=[ - util.ChoiceStringParameter( - name="falterBranch", - label="falter release branch", - choices=config['builter_releaseBranches'], - default=config['builter_releaseBranches'][0], - strict=True)], - reason=util.FixedParameter(name="reason", default="manual", hide=True))) + c["schedulers"].append( + schedulers.ForceScheduler( + name="force-release", + label="Release", + buttonName="Build Release", + builderNames=["builds/targets"], + codebases=[ + util.CodebaseParameter( + "", + label="Build falter release", + branch=util.ChoiceStringParameter( + name="branch", + label="git branch", + choices=config["builter_branches"], + default=config["builter_branches"][0], + strict=True, + ), + revision=util.FixedParameter(name="revision", default=""), + repository=util.FixedParameter( + name="repository", default=config["builter_repo"] + ), + project=util.FixedParameter(name="project", default=""), + ) + ], + properties=[ + util.StringParameter( + name="falterVersion", + label="falter release version - e.g. 1.2.3 or 1.1.1-rc1", + ) + ], + reason=util.FixedParameter(name="reason", default="manual", hide=True), + ) + ) - c['schedulers'].append(schedulers.ForceScheduler( - name="force-release", - label="Release", - buttonName="Build Release", - builderNames=["builds/targets"], - codebases=[ - util.CodebaseParameter( - "", - label="Build falter release", - branch=util.ChoiceStringParameter( - name="branch", - label="git branch", - choices=config['builter_branches'], - default=config['builter_branches'][0], - strict=True), - revision=util.FixedParameter(name="revision", default=""), - repository=util.FixedParameter(name="repository", default=config['builter_repo']), - project=util.FixedParameter(name="project", default=""))], - properties=[ - util.StringParameter( - name="falterVersion", - label="falter release version - e.g. 1.2.3 or 1.1.1-rc1")], - reason=util.FixedParameter(name="reason", default="manual", hide=True))) + c["builders"].append( + util.BuilderConfig( + name="builds/targets", + workernames=["masterworker"], + factory=targetsFactory(util.BuildFactory(), config["publishDir"]), + collapseRequests=False, + ) + ) - c['builders'].append(util.BuilderConfig( - name="builds/targets", - workernames=["masterworker"], - factory=targetsFactory(util.BuildFactory(), config['publishDir']), - collapseRequests=False)) + c["builders"].append( + util.BuilderConfig( + name="dummy/targets", + workernames=config["workerNames"], + factory=targetsTargetFactory( + util.BuildFactory(), + config["publishDir"], + config["publishURL"], + config["alpineVersion"], + ), + collapseRequests=False, + ) + ) - c['builders'].append(util.BuilderConfig( - name="dummy/targets", - workernames=config['workerNames'], - factory=targetsTargetFactory(util.BuildFactory(), config['publishDir'], config['publishURL'], config['alpineVersion']), - collapseRequests=False)) + return c - return c # Passed by targetsFactory to AsyncBuildGenerator to be called for each arch. def targetTriggerStep(target): - return AsyncTrigger( - # Step name limit is 50 chars, longest is currently 26 chars: - # "trigger lantiq/xway_legacy" - # See https://github.com/buildbot/buildbot/issues/3413 - name=util.Interpolate("trigger %(kw:target)s", target=target), - waitForFinish=True, - warnOnFailure=True, - schedulerNames=["dummy/targets"], - copy_properties=['repository', 'branch', 'revision', 'got_revision', 'falterBranch', 'falterVersion'], - set_properties={ - 'target': target, - 'origbuildnumber': util.Interpolate("%(prop:buildnumber)s"), - # Builder name limit is 70 characters, longest is currently 40 chars: - # targets/openwrt-22.03/lantiq/xway_legacy - # See https://github.com/buildbot/buildbot/pull/3957 - 'virtual_builder_name': util.Interpolate("targets/%(prop:falterBranch)s/%(kw:target)s", target=target), - 'virtual_builder_tags': ["targets", util.Interpolate("%(prop:falterBranch)s")]}) + return AsyncTrigger( + # Step name limit is 50 chars, longest is currently 26 chars: + # "trigger lantiq/xway_legacy" + # See https://github.com/buildbot/buildbot/issues/3413 + name=util.Interpolate("trigger %(kw:target)s", target=target), + waitForFinish=True, + warnOnFailure=True, + schedulerNames=["dummy/targets"], + copy_properties=[ + "repository", + "branch", + "revision", + "got_revision", + "falterBranch", + "falterVersion", + ], + set_properties={ + "target": target, + "origbuildnumber": util.Interpolate("%(prop:buildnumber)s"), + # Builder name limit is 70 characters, longest is currently 40 chars: + # targets/openwrt-22.03/lantiq/xway_legacy + # See https://github.com/buildbot/buildbot/pull/3957 + "virtual_builder_name": util.Interpolate( + "targets/%(prop:falterBranch)s/%(kw:target)s", target=target + ), + "virtual_builder_tags": [ + "targets", + util.Interpolate("%(prop:falterBranch)s"), + ], + }, + ) + @util.renderer def targetsFalterBranch(props): - fb = props.getProperty("falterBranch", "") - fv = props.getProperty("falterVersion", "") - if fb != "": - return fb - elif fv.startswith("1.2.3"): - return "1.2.3-snapshot" - elif fv.startswith("1.3.0"): - return "1.3.0-snapshot" - elif fv.startswith("1.4.0"): - return "1.4.0-snapshot" - else: - return "snapshot" + fb = props.getProperty("falterBranch", "") + fv = props.getProperty("falterVersion", "") + if fb != "": + return fb + elif fv.startswith("1.2.3"): + return "1.2.3-snapshot" + elif fv.startswith("1.3.0"): + return "1.3.0-snapshot" + elif fv.startswith("1.4.0"): + return "1.4.0-snapshot" + else: + return "snapshot" + @util.renderer def targetsFalterVersion(props): - return props.getProperty("falterVersion", props.getProperty("falterBranch", "")) + return props.getProperty("falterVersion", props.getProperty("falterBranch", "")) + @util.renderer def targetsPubDir(props): - fv = props.getProperty("falterVersion", "") - if '-' in fv or 'snapshot' in fv or 'testbuildbot' in fv: - return "unstable/"+fv - else: - return "stable/"+fv + fv = props.getProperty("falterVersion", "") + if "-" in fv or "snapshot" in fv or "testbuildbot" in fv: + return "unstable/" + fv + else: + return "stable/" + fv + # Fans out to one builder per target and blocks for the results. def targetsFactory(f, wwwPrefix): @@ -130,12 +179,16 @@ def targetsFactory(f, wwwPrefix): steps.SetProperty( name=util.Interpolate("falterBranch = %(kw:fb)s", fb=targetsFalterBranch), property="falterBranch", - value=targetsFalterBranch)) + value=targetsFalterBranch, + ) + ) f.addStep( steps.SetProperty( name=util.Interpolate("falterVersion = %(kw:fv)s", fv=targetsFalterVersion), property="falterVersion", - value=targetsFalterVersion)) + value=targetsFalterVersion, + ) + ) f.addStep( steps.Git( name="git clone", @@ -143,24 +196,30 @@ def targetsFactory(f, wwwPrefix): repourl=util.Interpolate("%(prop:repository)s"), # if using incremental, we get strange behaviors, when configuring # another repo, that is back the other repo. Doing full checkouts is cleaner - method='clobber', - mode='full', - submodules=True)) + method="clobber", + mode="full", + submodules=True, + ) + ) f.addStep( - AsyncBuildGenerator(targetTriggerStep, + AsyncBuildGenerator( + targetTriggerStep, name="generate builds", haltOnFailure=True, - command=["sh", "-c", util.Interpolate( - # Read our list of targets to build. - # - # 1. Print the architectures-to-targets mapping - # 2. Remove comments and empty lines - # 3. Take everything but the first line - # 4. Print all entries into $targets one by one - # 5. Go through $targets, print only targets that aren't broken - # - # TODO: doesn't fail if targets-*.txt doesn't exist - '''\ + command=[ + "sh", + "-c", + util.Interpolate( + # Read our list of targets to build. + # + # 1. Print the architectures-to-targets mapping + # 2. Remove comments and empty lines + # 3. Take everything but the first line + # 4. Print all entries into $targets one by one + # 5. Go through $targets, print only targets that aren't broken + # + # TODO: doesn't fail if targets-*.txt doesn't exist + """\ targets=$(\ cat .buildconf/targets-%(prop:falterBranch)s.txt \ | grep -v "#" | grep . \ @@ -173,42 +232,49 @@ def targetsFactory(f, wwwPrefix): echo "$t" ; \ fi ; \ done \ -''')])) +""" + ), + ], + ) + ) wwwdir = util.Interpolate( - "%(kw:prefix)s/builds/targets/%(prop:buildnumber)s", prefix=wwwPrefix) - pubdir = util.Interpolate( - "%(kw:pr)s/%(kw:pd)s", pr=wwwPrefix, pd=targetsPubDir) + "%(kw:prefix)s/builds/targets/%(prop:buildnumber)s", prefix=wwwPrefix + ) + pubdir = util.Interpolate("%(kw:pr)s/%(kw:pd)s", pr=wwwPrefix, pd=targetsPubDir) f.addStep( steps.MasterShellCommand( name="publish", haltOnFailure=True, - command=["sh", "-c", util.Interpolate( - # Publish build in a way that minimizes downtime. - # - # We call the currently published artifacts "current", - # which is e.g. /unstable/1.3.0-snapshot or /unstable/snapshot. - # We also use two temporary directories called "new" and "prev". - # - # 1. Remove "new" and "prev" leftovers from previous builds - # 2. Move build artifacts into "new" - # 3. Rename "current" published stuff to "prev" - # 4. Publish "new" by renaming it to "current" - # 5. Remove "prev" - # - # This is slightly different from packages builds, - # where we copy+move instead of just move. - # We just don't have the hardware to copy >50 GB quickly. - # That means /builds/targets/%d is empty after publishing, - # while it's kept available for packages builds. - # - # Targets downloads are only unavailable after step 3 and - # before step 4 has completed. - # - # Just symlinking from /builds/targets/%d is not a good option, - # since we want to be able to just delete that at any time, - # without worrying about symlinks pointing to deleted stuff. - """\ + command=[ + "sh", + "-c", + util.Interpolate( + # Publish build in a way that minimizes downtime. + # + # We call the currently published artifacts "current", + # which is e.g. /unstable/1.3.0-snapshot or /unstable/snapshot. + # We also use two temporary directories called "new" and "prev". + # + # 1. Remove "new" and "prev" leftovers from previous builds + # 2. Move build artifacts into "new" + # 3. Rename "current" published stuff to "prev" + # 4. Publish "new" by renaming it to "current" + # 5. Remove "prev" + # + # This is slightly different from packages builds, + # where we copy+move instead of just move. + # We just don't have the hardware to copy >50 GB quickly. + # That means /builds/targets/%d is empty after publishing, + # while it's kept available for packages builds. + # + # Targets downloads are only unavailable after step 3 and + # before step 4 has completed. + # + # Just symlinking from /builds/targets/%d is not a good option, + # since we want to be able to just delete that at any time, + # without worrying about symlinks pointing to deleted stuff. + """\ mkdir -p %(kw:p)s %(kw:p)s.new \ && rm -rf %(kw:p)s.new/* %(kw:p)s.prev \ && mv %(kw:w)s/* %(kw:p)s.new/ \ @@ -216,46 +282,56 @@ def targetsFactory(f, wwwPrefix): && mv %(kw:p)s.new %(kw:p)s \ && rm -rf %(kw:p)s.prev \ """, - w=wwwdir, p=pubdir)])) + w=wwwdir, + p=pubdir, + ), + ], + ) + ) return f + @util.renderer def targetsTarFile(props): - t, st = props['target'].split('/') - return "targets-{0}-{1}_{2}.tar".format(props['origbuildnumber'], t, st) + t, st = props["target"].split("/") + return "targets-{0}-{1}_{2}.tar".format(props["origbuildnumber"], t, st) + def targetsTargetFactory(f, wwwPrefix, wwwURL, alpineVersion): f.addStep( steps.ShellCommand( name="build", haltOnFailure=True, - interruptSignal='TERM', # podman can't proxy the default KILL signal - command=["sh", "-c", util.Interpolate( - # Container that prints a tarball to stdout, and other output to stderr. - # - # Parameters: - # * -i so we get output - # * no -t because it messes with stdout/stderr - # * --rm so we don't fill up the disk with old containers - # * --log-driver so we don't pump huge tarballs into the logging facility - # * it'd also trigger a segfault in ~15% of concurrent runs: - # https://github.com/containers/podman/issues/13779 - # - # Originally we'd also set --timeout, but the workers perf is - # very inconsistent. Build times regularly went very long, - # so long that --timeout didn't make sense anymore. - # It was more of a preventive measure anyway, - # since hanging builds aren't a problem that we've had so far. - # In addition, manual build cancellation is very reliable - # and would be an effective countermeasure. - # - # We tried to speed things up with a ramdisk filesystem, - # but this had very little performance impact in production: - # --tmpfs /root:rw,size=6291456k,mode=1777 - # Also larger targets seemed to need more than 6 GiB. - # - """\ + interruptSignal="TERM", # podman can't proxy the default KILL signal + command=[ + "sh", + "-c", + util.Interpolate( + # Container that prints a tarball to stdout, and other output to stderr. + # + # Parameters: + # * -i so we get output + # * no -t because it messes with stdout/stderr + # * --rm so we don't fill up the disk with old containers + # * --log-driver so we don't pump huge tarballs into the logging facility + # * it'd also trigger a segfault in ~15% of concurrent runs: + # https://github.com/containers/podman/issues/13779 + # + # Originally we'd also set --timeout, but the workers perf is + # very inconsistent. Build times regularly went very long, + # so long that --timeout didn't make sense anymore. + # It was more of a preventive measure anyway, + # since hanging builds aren't a problem that we've had so far. + # In addition, manual build cancellation is very reliable + # and would be an effective countermeasure. + # + # We tried to speed things up with a ramdisk filesystem, + # but this had very little performance impact in production: + # --tmpfs /root:rw,size=6291456k,mode=1777 + # Also larger targets seemed to need more than 6 GiB. + # + """\ podman run -i --rm --log-driver=none docker.io/library/alpine:%(kw:alpineVersion)s sh -c '\ ( \ apk add git bash wget xz gzip unzip grep diffutils findutils coreutils build-base gcc abuild binutils ncurses-dev gawk bzip2 gettext perl python3 rsync sqlite flex libxslt \ @@ -268,11 +344,18 @@ def targetsTargetFactory(f, wwwPrefix, wwwURL, alpineVersion): ) >&2 \ && cd /root/falter-builter/firmwares \ && tar -c *' > out.tar \ -""", alpineVersion=alpineVersion)])) +""", + alpineVersion=alpineVersion, + ), + ], + ) + ) tarfile = targetsTarFile wwwpath = util.Interpolate("builds/targets/%(prop:origbuildnumber)s/") - wwwdir = util.Interpolate("%(kw:prefix)s/%(kw:wwwpath)s", prefix=wwwPrefix, wwwpath=wwwpath) + wwwdir = util.Interpolate( + "%(kw:prefix)s/%(kw:wwwpath)s", prefix=wwwPrefix, wwwpath=wwwpath + ) wwwurl = util.Interpolate("%(kw:url)s/%(kw:wwwpath)s", url=wwwURL, wwwpath=wwwpath) f.addStep( steps.FileUpload( @@ -281,27 +364,43 @@ def targetsTargetFactory(f, wwwPrefix, wwwURL, alpineVersion): workersrc="out.tar", masterdest=tarfile, url=wwwurl, - urlText=wwwurl)) + urlText=wwwurl, + ) + ) f.addStep( steps.MasterShellCommand( name="extract", haltOnFailure=True, - command=["sh", "-c", util.Interpolate( - "mkdir -vp %(kw:wwwdir)s && tar -v -C %(kw:wwwdir)s -xf %(kw:tarfile)s", - tarfile=tarfile, wwwdir=wwwdir)])) + command=[ + "sh", + "-c", + util.Interpolate( + "mkdir -vp %(kw:wwwdir)s && tar -v -C %(kw:wwwdir)s -xf %(kw:tarfile)s", + tarfile=tarfile, + wwwdir=wwwdir, + ), + ], + ) + ) f.addStep( steps.MasterShellCommand( name="cleanup master", alwaysRun=True, warnOnFailure=False, - command=["sh", "-c", util.Interpolate( - "rm -vf %(kw:tarfile)s", - tarfile=tarfile)])) + command=[ + "sh", + "-c", + util.Interpolate("rm -vf %(kw:tarfile)s", tarfile=tarfile), + ], + ) + ) f.addStep( steps.ShellCommand( name="cleanup worker", alwaysRun=True, warnOnFailure=False, - command=["sh", "-c", "rm -vf out.tar"])) + command=["sh", "-c", "rm -vf out.tar"], + ) + ) return f diff --git a/roles/ff_monitor/files/create_node_list.py b/roles/ff_monitor/files/create_node_list.py index 1f1723d..aa4da0f 100644 --- a/roles/ff_monitor/files/create_node_list.py +++ b/roles/ff_monitor/files/create_node_list.py @@ -1,19 +1,20 @@ #!/bin/env python3 -import urllib.request,json +import json +import urllib.request with urllib.request.urlopen("https://hopglass.berlin.freifunk.net/nodes.json") as url: data = json.loads(url.read().decode()) simplenodelist = list() - nodes = data.get('nodes') + nodes = data.get("nodes") for node in nodes: simplenode = { - "latitude": node['nodeinfo']['location']['latitude'], - "longitude": node['nodeinfo']['location']['longitude'], - "name": node['nodeinfo']['hostname'], - "key": node['nodeinfo']['hostname'] + "latitude": node["nodeinfo"]["location"]["latitude"], + "longitude": node["nodeinfo"]["location"]["longitude"], + "name": node["nodeinfo"]["hostname"], + "key": node["nodeinfo"]["hostname"], } simplenodelist.append(simplenode.copy()) - print (json.dumps(simplenodelist, separators=(',', ':'))) + print(json.dumps(simplenodelist, separators=(",", ":"))) From 424d7c002885f33c9f32beab71115e79fbebfd44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20H=C3=BCbner?= Date: Wed, 20 Sep 2023 12:30:54 +0200 Subject: [PATCH 09/17] targets.py: silence jscpd at config section. --- roles/buildbot/files/targets.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/roles/buildbot/files/targets.py b/roles/buildbot/files/targets.py index ac7c8ef..277c1da 100644 --- a/roles/buildbot/files/targets.py +++ b/roles/buildbot/files/targets.py @@ -5,6 +5,11 @@ from buildbot.plugins import * +# linter should not bother us with variable "schedulers" not defined. It it +# defined indeed and is an imported object +# pylint: disable=E0602 + +# jscpd:ignore-start def targetsConfig(c, config): c["schedulers"].append( schedulers.Triggerable(name="dummy/targets", builderNames=["dummy/targets"]) @@ -105,7 +110,7 @@ def targetsConfig(c, config): ) return c - +# jscpd:ignore-end # Passed by targetsFactory to AsyncBuildGenerator to be called for each arch. def targetTriggerStep(target): From 0639cfcf51aa41e08089dd8c2d31a6f4472b0fb9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20H=C3=BCbner?= Date: Wed, 20 Sep 2023 13:02:55 +0200 Subject: [PATCH 10/17] targets.py: deactivate flake8 morning for too long lines --- roles/buildbot/files/targets.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/roles/buildbot/files/targets.py b/roles/buildbot/files/targets.py index 277c1da..2386015 100644 --- a/roles/buildbot/files/targets.py +++ b/roles/buildbot/files/targets.py @@ -4,6 +4,10 @@ from asyncbuild import * from buildbot.plugins import * +# shut up, flake8 and learn to deal with bulk imports instead of loading everthing +# explicitly! We don't bother for lines longer than 80 chars too. We don't use +# punchcards anymore! +# flake8: noqa: F403, F405, E501 # linter should not bother us with variable "schedulers" not defined. It it # defined indeed and is an imported object From 959fe65cca32ddb570e654c329b8b65a1ff1c802 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20H=C3=BCbner?= Date: Wed, 20 Sep 2023 13:03:46 +0200 Subject: [PATCH 11/17] packages.py: silence flake8 roaring for lines longer than 80 chars --- roles/buildbot/files/packages.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/roles/buildbot/files/packages.py b/roles/buildbot/files/packages.py index 0cba7b1..f9ae224 100644 --- a/roles/buildbot/files/packages.py +++ b/roles/buildbot/files/packages.py @@ -1,11 +1,17 @@ # -*- python -*- # ex: set filetype=python: -import re - from asyncbuild import * from buildbot.plugins import * +# shut up, flake8 and learn to deal with bulk imports instead of loading everthing +# explicitly! We don't bother for lines longer than 80 chars too. We don't use +# punchcards anymore! +# flake8: noqa: F403, F405, E501 + +# linter should not bother us with variable "schedulers" not defined. It it +# defined indeed and is an imported object +# pylint: disable=E0602 def packagesConfig(c, config): c["schedulers"].append( From d9dfcc1df105d65d5da121552f4e28617b2859e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20H=C3=BCbner?= Date: Wed, 20 Sep 2023 13:06:11 +0200 Subject: [PATCH 12/17] python-files: Apply black formatter (adds blank lines only) --- roles/buildbot/files/packages.py | 1 + roles/buildbot/files/targets.py | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/roles/buildbot/files/packages.py b/roles/buildbot/files/packages.py index f9ae224..d30bb72 100644 --- a/roles/buildbot/files/packages.py +++ b/roles/buildbot/files/packages.py @@ -13,6 +13,7 @@ # defined indeed and is an imported object # pylint: disable=E0602 + def packagesConfig(c, config): c["schedulers"].append( schedulers.Triggerable(name="dummy/packages", builderNames=["dummy/packages"]) diff --git a/roles/buildbot/files/targets.py b/roles/buildbot/files/targets.py index 2386015..0d1f1da 100644 --- a/roles/buildbot/files/targets.py +++ b/roles/buildbot/files/targets.py @@ -13,6 +13,7 @@ # defined indeed and is an imported object # pylint: disable=E0602 + # jscpd:ignore-start def targetsConfig(c, config): c["schedulers"].append( @@ -114,8 +115,11 @@ def targetsConfig(c, config): ) return c + + # jscpd:ignore-end + # Passed by targetsFactory to AsyncBuildGenerator to be called for each arch. def targetTriggerStep(target): return AsyncTrigger( From ea793642a01776ff163f1897c279d72ae00a4525 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20H=C3=BCbner?= Date: Wed, 20 Sep 2023 13:30:40 +0200 Subject: [PATCH 13/17] fix nitpicking linters in notify.py --- roles/buildbot/files/notify.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/roles/buildbot/files/notify.py b/roles/buildbot/files/notify.py index 3933004..572fc21 100644 --- a/roles/buildbot/files/notify.py +++ b/roles/buildbot/files/notify.py @@ -1,9 +1,8 @@ # -*- python -*- # ex: set filetype=python: -import pprint -from buildbot.plugins import steps, util +from buildbot.plugins import util from buildbot.process.results import statusToString from buildbot.reporters.base import ReporterBase from buildbot.reporters.generators.build import BuildStartEndStatusGenerator @@ -11,7 +10,6 @@ from buildbot.reporters.message import MessageFormatterRenderable from buildbot.util import httpclientservice from twisted.internet import defer -from twisted.python import log class MatrixNotifier(ReporterBase): @@ -64,7 +62,7 @@ def sendMessage(self, reports): if ( "number" not in b - or b["complete"] != True + or b["complete"] is not True or (builder != "builds/targets" and builder != "builds/packages") ): return @@ -84,16 +82,14 @@ def sendMessage(self, reports): url = b["url"] msg = f"{builder} @ {version} · {result} · {url}" - htmlmsg = ( - f'{builder} @ {version} · {result} · {url}' - ) + htmlmsg = f'{builder} @ {version} · {result} · {url}' # noqa: E501 http = yield httpclientservice.HTTPClientService.getService( self.master, self.homeserver ) - res = http.post( - f"/_matrix/client/r0/rooms/{self.room}/send/m.room.message?access_token={self.accessToken}", + http.post( + f"/_matrix/client/r0/rooms/{self.room}/send/m.room.message?access_token={self.accessToken}", # noqa: E501 json={ "msgtype": "m.notice", "body": msg, From 72ece5688241ac07110287bcd955d15644ce5c9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20H=C3=BCbner?= Date: Wed, 20 Sep 2023 13:31:19 +0200 Subject: [PATCH 14/17] fix nitpicking linters in asyncbuild.py --- roles/buildbot/files/asyncbuild.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/roles/buildbot/files/asyncbuild.py b/roles/buildbot/files/asyncbuild.py index 08d4436..98455cf 100644 --- a/roles/buildbot/files/asyncbuild.py +++ b/roles/buildbot/files/asyncbuild.py @@ -4,11 +4,15 @@ from typing import List from buildbot.plugins import steps, util -from buildbot.process import build, buildstep, factory, logobserver +from buildbot.process import build, buildstep, logobserver from twisted.internet import defer from twisted.python import log +# We don't care for "too long" lines. We don't use punchcards anymore! +# flake8: noqa + + # AsyncBuildGenerator dynamically generates build steps from command output. # # It runs stepFunc for every line of stdout from specified command. From 14597ff636ababba23464168056f2a48bbcb6a28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20H=C3=BCbner?= Date: Wed, 20 Sep 2023 13:34:44 +0200 Subject: [PATCH 15/17] fix nitpicking linters in generate_autoupdate.py --- roles/buildbot/files/generate_autoupdate_json.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/roles/buildbot/files/generate_autoupdate_json.py b/roles/buildbot/files/generate_autoupdate_json.py index c9f5578..df0c193 100644 --- a/roles/buildbot/files/generate_autoupdate_json.py +++ b/roles/buildbot/files/generate_autoupdate_json.py @@ -6,6 +6,9 @@ import json import os +# We don't care for "too long" lines. We don't use punchcards anymore! +# flake8: noqa + parser = argparse.ArgumentParser() parser.add_argument( "-v", @@ -52,7 +55,7 @@ # get target and create dict, if not already created target = "/".join(fpath.split("/")[-3:-1]) - if autoupdate_json.get("target").get(target) == None: + if autoupdate_json.get("target").get(target) is None: autoupdate_json["target"][target] = {} # get hashsums @@ -65,7 +68,7 @@ break # add information to autoupdate.json - if autoupdate_json.get("target").get(target).get(model) == None: + if autoupdate_json.get("target").get(target).get(model) is None: autoupdate_json["target"][target][model] = {} if sha256 is not None: From a56f19fabeaca3ab05f20f2b30b5eb429166d3fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20H=C3=BCbner?= Date: Wed, 20 Sep 2023 13:45:52 +0200 Subject: [PATCH 16/17] fix nitpicking linters in generate_autoupdate.py, mypy edition --- roles/buildbot/files/generate_autoupdate_json.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/roles/buildbot/files/generate_autoupdate_json.py b/roles/buildbot/files/generate_autoupdate_json.py index df0c193..297349f 100644 --- a/roles/buildbot/files/generate_autoupdate_json.py +++ b/roles/buildbot/files/generate_autoupdate_json.py @@ -2,6 +2,9 @@ # {{ ansible_managed }} +# silence mypy +# type: ignore + import argparse import json import os From c0dcb9cb6723f612dd9aa23eba643ff715b8a591 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20H=C3=BCbner?= Date: Wed, 20 Sep 2023 13:48:10 +0200 Subject: [PATCH 17/17] fix nitpicking linters in asyncbuild.py, isort edition --- roles/buildbot/files/asyncbuild.py | 1 - 1 file changed, 1 deletion(-) diff --git a/roles/buildbot/files/asyncbuild.py b/roles/buildbot/files/asyncbuild.py index 98455cf..7e28b98 100644 --- a/roles/buildbot/files/asyncbuild.py +++ b/roles/buildbot/files/asyncbuild.py @@ -8,7 +8,6 @@ from twisted.internet import defer from twisted.python import log - # We don't care for "too long" lines. We don't use punchcards anymore! # flake8: noqa