diff --git a/.bumpversion.cfg b/.bumpversion.cfg index db0f4896..a7f214c1 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 4.3.19 +current_version = 4.3.29 commit = True tag = False diff --git a/LICENSE b/LICENSE index 45ebf28c..90939cb4 100644 --- a/LICENSE +++ b/LICENSE @@ -2,7 +2,8 @@ Version 2.0, January 2004 http://www.apache.org/licenses/ - Copyright 2017 Gregor von Laszewski, Indiana University + Copyright 2017-2021 Gregor von Laszewski, Indiana University + Copyright 2021,2022 Gregor von Laszewski, University of Virginia Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/README.md b/README.md index 99a76734..30b385bf 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,5 @@ # Cloudmesh Pi Burner for SD Cards -**WARNING:** *This program is designed for a **Raspberry Pi**. Instructions -to use **Linux** and macOS are included in the FAQ. If you want to help us port -to any other OSes, such as Windows 10, please contact laszewski@gmail.com* - [![image](https://travis-ci.com/cloudmesh/cloudmesh-pi-burn.svg?branch=main)](https://travis-ci.com/github/cloudmesh/cloudmesh-pi-burn) [![image](https://img.shields.io/pypi/pyversions/cloudmesh-pi-burn.svg)](https://pypi.org/project/cloudmesh-pi-burn) [![image](https://img.shields.io/pypi/v/cloudmesh-pi-burn.svg)](https://pypi.org/project/cloudmesh-pi-burn/) @@ -13,495 +9,49 @@ to any other OSes, such as Windows 10, please contact laszewski@gmail.com* - [Cloudmesh Pi Burner for SD Cards](#cloudmesh-pi-burner-for-sd-cards) - - [1. Introduction](#1-introduction) - - [2. Nomenclature](#2-nomenclature) - - [3. Quickstarts](#3-quickstarts) - - [4. Setup for Bridged WiFi](#4-setup-for-bridged-wifi) - - [4.1 Requirements](#41-requirements) - - [4.2 Manager Pi](#42-manager-pi) - - [4.3 Burning Multiple SD Cards with a Single Burner](#43-burning-multiple-sd-cards-with-a-single-burner) - - [4.4 Connecting Pis to the Internet via Bridge](#44-connecting-pis-to-the-internet-via-bridge) - - [5. Set up of the SSH keys and SSH tunnel](#5-set-up-of-the-ssh-keys-and-ssh-tunnel) - - [6. Manual Pages](#6-manual-pages) + - [1. Burning Tutorials](#1-introduction) + - [2. Manual Pages](#6-manual-pages) - [6.1 Manual Page for the `burn` command](#61-manual-page-for-the-burn-command) - [6.2 Manual Page for the `bridge` command](#62-manual-page-for-the-bridge-command) - [6.3 Manual Page for the `host` command](#63-manual-page-for-the-host-command) - [6.4 Manual Page for the `pi` command](#64-manual-page-for-the-pi-command) - [6.4 Manual Page for the `ssh` command](#64-manual-page-for-the-ssh-command) - - [7. FAQ and Hints](#7-faq-and-hints) - - [7.1 Quickstart for a Setup of a cluster from macOS or Linux with no burning on a PI.](#71-quickstart-for-a-setup-of-a-cluster-from-macos-or-linux-with-no-burning-on-a-pi) - - [7.2 Quickstart for Using a Pi to Burn a Cluster Using Inventory](#72-quickstart-for-using-a-pi-to-burn-a-cluster-using-inventory) - - [7.3 Can I use the LEDs on the PI Motherboard?](#73-can-i-use-the-leds-on-the-pi-motherboard) - - [7.4 How can I use pycharm, to edit files or access files in general from my Laptop on the PI?](#74-how-can-i-use-pycharm-to-edit-files-or-access-files-in-general-from-my-laptop-on-the-pi) - - [7.5 How can I enhance the `get` script?](#75-how-can-i-enhance-the-get-script) - - [7.6 Can I use a Mesh Network for the setup?](#76-can-i-use-a-mesh-network-for-the-setup) - - [7.7 Can I use cms burn on Linux?](#77-can-i-use-cms-burn-on-linux) - - [7.8 What packages do I need to run the info command on macOS](#78-what-packages-do-i-need-to-run-the-info-command-on-macos) - - [7.9 Are there any unit tests?](#79-are-there-any-unit-tests) - - [7.10 Using Pi Imager to setup a Manager Pi with headless access](#710-using-pi-imager-to-setup-a-manager-pi-with-headless-access) - - [7.11 Single Card Burning](#711-single-card-burning) - - [7.12 How to update firmware?](#712-how-to-update-firmware) - - [7.13 Alternatives](#713-alternatives) - - [7.14 How do I scann for WIFI networks?](#714-how-do-i-scann-for-wifi-networks) - - [7.15 What is the status of the implementation?](#715-what-is-the-status-of-the-implementation) - - [7.16 I run into a Kernal Panic on my burned Pi. What do I do?](#716-i-run-into-a-kernal-panic-on-my-burned-pi-what-do-i-do) - - [7.17 How do I enable password login?](#717-how-do-i-enable-password-login) - - [7.18 Becuase I am using and sd card extender, I need to set a cmdline argument to force 3.3V SD card operation.](#718-becuase-i-am-using-and-sd-card-extender-i-need-to-set-a-cmdline-argument-to-force-33v-sd-card-operation) - - [8. How can I contribute Contributing](#8-how-can-i-contribute-contributing) + - [3. FAQ and Hints](#7-faq-and-hints) + - [3.1 Quickstart for a Setup of a cluster from macOS or Linux with no burning on a PI.](#71-quickstart-for-a-setup-of-a-cluster-from-macos-or-linux-with-no-burning-on-a-pi) + - [3.2 Quickstart for Using a Pi to Burn a Cluster Using Inventory](#72-quickstart-for-using-a-pi-to-burn-a-cluster-using-inventory) + - [3.3 Can I use the LEDs on the PI Motherboard?](#73-can-i-use-the-leds-on-the-pi-motherboard) + - [3.4 How can I use pycharm, to edit files or access files in general from my Laptop on the PI?](#74-how-can-i-use-pycharm-to-edit-files-or-access-files-in-general-from-my-laptop-on-the-pi) + - [3.5 How can I enhance the `get` script?](#75-how-can-i-enhance-the-get-script) + - [3.6 Can I use a Mesh Network for the setup?](#76-can-i-use-a-mesh-network-for-the-setup) + - [3.7 Can I use cms burn on Linux?](#77-can-i-use-cms-burn-on-linux) + - [3.8 What packages do I need to run the info command on macOS](#78-what-packages-do-i-need-to-run-the-info-command-on-macos) + - [3.9 Are there any unit tests?](#79-are-there-any-unit-tests) + - [3.10 Using Pi Imager to setup a Manager Pi with headless access](#710-using-pi-imager-to-setup-a-manager-pi-with-headless-access) + - [3.11 Single Card Burning](#711-single-card-burning) + - [3.12 How to update firmware?](#712-how-to-update-firmware) + - [3.13 Alternatives](#713-alternatives) + - [3.14 How do I scann for WIFI networks?](#714-how-do-i-scann-for-wifi-networks) + - [3.15 What is the status of the implementation?](#715-what-is-the-status-of-the-implementation) + - [3.16 I run into a Kernal Panic on my burned Pi. What do I do?](#716-i-run-into-a-kernal-panic-on-my-burned-pi-what-do-i-do) + - [3.17 How do I enable password login?](#717-how-do-i-enable-password-login) + - [3.18 How do I use SDCard externers with different voltage?](#718-how-do-i-use-sdcard-externers-with-different-voltage) + - [3.19 How do I get the latest image if a new image was released?](#719-how-do-i-get-the-latest-image-if-a-new-image-was-released) + - [4. How can I contribute Contributing](#8-how-can-i-contribute-contributing) -## 1. Introduction - -`cms burn` is a program to burn many SD cards for the preparation of -building clusters with Raspberry Pi's. It allows users to create -readily bootable SD cards that have the network configured and contain a -public ssh key from your machine that you then use to login into the -PIs after boot. Thus, little setup is needed for a cluster after boot. Another -unique feature is that you can burn multiple cards in a row, each with -their individual setup such as hostnames and ipadresses. - - -## 2. Nomenclature - -* Commands proceeded with `pi@managerpi:$` are to be executed on the - Raspberry Pi with the name `managerpi`. - -* Commands with `(ENV3) pi@managerpi:$` are to be executed in a virtual ENV - using Python 3 on the Raspberry Pi with the name managerpi - -## 3. Quickstarts - -We provide the following quickstarts: - -1. [Quickstart to burn worker SD Cards on a PI via an inventory](#72-quickstart-for-using-a-pi-to-burn-a-cluster-using-inventory) -2. [Quickstart to burn all cards on macOS or Linux vi the cluster command](#71-quickstart-for-a-setup-of-a-cluster-from-macos-or-linux-with-no-burning-on-a-pi) - - On macOS this requires you have write access for ext4 which requires the - purchase of additional software. If you do not lke to do this see the - solution discussed in 1. - -However before you do the Quickstarts we encourage you to look at other -features we provide. - -## 4. Setup for Bridged WiFi - -To provide you with a glimpse of what you can do with cms burn, we -have provided this quickstart guide that will create one manager PI and -several workers. - -This setup is intended for those who have restricted access to their -network (ie. cannot access router controls). For example, those -on campus WiFis or regulated apartment WiFis. - -Figure 1 describes our network configuration. We have 5 -Raspberry Pi 4s: 1 manager and 4 workers. We have WiFi access, but we -do not necessarily have wired access or access to the router's controls. - -We also have a network switch, where the manager and workers can -communicate locally, but we will configure the manager to provide -internet access to devices on the network switch via a "network -bridge". - -![](https://github.com/cloudmesh/cloudmesh-pi-burn/raw/main/images/network-bridge.png) - -Figure 1: Pi Cluster setup with bridge network - -### 4.1 Requirements - -For the quickstart we have the following hardware requirements: - -* SD Cards and Raspberry Pis - -* You will need an SD card reader/writer (USB-A) to burn new cards. We - recommend that you buy a USB 3.0 SDCard reader/writer as they are - significantly faster and you can reuse them on PI'4s. Make sure to - get an adapter if your normal computer only supports USB-C - -* You will need a Raspberry Pi cluster. Detailed information about - parts are provided at our - [Web page](https://cloudmesh.github.io/pi/docs/hardware/parts/). - -### 4.2 Manager Pi - -First we need to configure the Manager Pi - -**Step 0.** Burn Manager Pi SD Card - -Using [Raspberry Pi imager](https://www.raspberrypi.org/software/), - burn an SD card with *Raspberry Pi OS (32-bit) with desktop and - recommended applications*. You may use your normal system to burn - such a card including Windows, macOS, or Linux. - -You will then want a method of accessing this manager Pi. You may -either use SSH (recommended) from the command line or a monitor to use the desktop environment (easiest) -to access it. We highly recommend -[changing the password](https://www.raspberrypi.org/documentation/linux/usage/users.md) -on the Pi as soon as you have access. This is because the pi is -initialized with default user `pi` and default password -`raspberry`. This is critical if you are on a shared network where -anyone can attempt to access your pi. - -> Monitor Desktop Environment: You will need a monitor, keyboard, and -> mouse. This is the easiest approach as Raspberry Pi OS provides a -> very nice user interface with an easy-to-follow setup process for -> connecting to WiFi and other such tasks. - -> SSH Environment: You may consider enabling SSH access to your Pi so -> that you may access the file system from your preferred machine. - -> Headless Configuration: See section 3 of -> [enabling ssh](https://www.raspberrypi.org/documentation/remote-access/ssh/) -> for instructions on how to enable SSH headlessly. Similarly, -> [how to enable WiFi headlessly](https://raspberrypi.stackexchange.com/questions/10251/prepare-sd-card-for-wifi-on-headless-pi). -> Additionally you can check out the FAQ for step-by-step -> instructions. -> [Using Pi Imager to setup a Manager Pi with headless access](#using-pi-imager-to-setup-a-manager-pi-with-headless-access). - -> Update the firmware: See the FAQ [How to update firmware?](#712-how-to-update-firmware) - -**Step 1.** Installing Cloudmesh on the Manager Pi - -Open a new terminal screen on the Manager Pi. Here we assume the -hostname is `managerpi`. However, this is of no importance in relation -to the topics of this guide. - -Update pip. The simple curl command below will generate an ssh-key, -update your system, and install cloudmesh. - -``` -pi@managerpi:~ $ pip install pip -U -pi@managerpi:~ $ curl -Ls http://cloudmesh.github.io/get/pi | sh - -``` - -This will take a moment... - -**Step 2.** Reboot - -The installation script updates your system. Reboot for effect. - -``` -pi@managerpi:~ $ sudo reboot -``` - -**Step 3.** Download the latest Raspberry Pi Lite OS - -The following command will download the latest images for Raspberry -Lite OS. - -``` -(ENV3) pi@managerpi:~ $ cms burn image get latest-lite -``` - -We can verify our image's downloaded with the following. - -``` -(ENV3) pi@managerpi:~ $ cms burn image ls -``` - -> **Note.** For our cluster we use light, but if you like -> to use other versions please see this note. -> We can use the following command to list the current -> Raspberry Pi OS versions (full and lite) -> -> ``` -> (ENV3) pi@managerpi:~ $ cms burn image versions --refresh -> ``` -> -> This will list the Tags and Types of each available OS. We can then -> modify the `image get` command for versions we are interested in. For -> example, -> -> ``` -> (ENV3) pi@managerpi:~ $ cms burn image get full-2020-05-28 -> ``` - - -**Step 4**. Setup SD Card Writer - -Plug your SD Card Writer into the Pi. Ensure you have an SD Card -inserted into your writer. Run the following command to find the path -to your SD Card. - -``` -(ENV3) pi@managerpi:~ $ cms burn info -... -# ---------------------------------------------------------------------- -# SD Cards Found -# ---------------------------------------------------------------------- - -+----------+----------------------+----------+-----------+-------+------------------+---------+-----------+-----------+ -| Path | Info | Readable | Formatted | Empty | Size | Aaccess | Removable | Writeable | -+----------+----------------------+----------+-----------+-------+------------------+---------+-----------+-----------+ -| /dev/sda | Generic Mass-Storage | True | True | False | 64.1 GB/59.7 GiB | True | True | | -+----------+----------------------+----------+-----------+-------+------------------+---------+-----------+-----------+ -``` - -> `cms burn info` has other useful information, but for the purposes -> of this guide we omit it. - -We can see from the information displayed that our SD card's path is -`/dev/sda`. Of course, this may vary. - -### 4.3 Burning Multiple SD Cards with a Single Burner - -**Step 0.** Ensure the first SD card is inserted into the burner. - -We can run `cms burn info` again as we did above to verify our SD -Card is connected. - -**Step 1.** Burning the Cards - -`cms burn` supports parameterized hostnames that allow automatic incrementation -of numbers. - -For example, `red00[1-2]` is interpreted by cms burn as `[red001, red002]`. -Similarly, `red[a-c]` is interpreted by cms burn as `[reda, redb, redc]`. - -We can burn 2 SD cards as follows: - -**!! WARNING VERIFY THE DEVICE IS CORRECT. REFER TO CMS BURN !!** - -``` -(ENV3) pi@managerpi:~ $ cms burn create --hostname=red00[1-4] --ip=10.1.1.[2-5] --device=/dev/sda --tag=latest-lite -``` - -The user will be prompted to swap the SD cards after each card burn if -there are still remaining cards to burn. - -**Step 2.** Boot the cluster - -After all cards are burned. Turn off the cluster, insert the cards, and turn -the cluster back on. - -We can now proceed to the next section where we configure our bridge. - -### 4.4 Connecting Pis to the Internet via Bridge - -Figure 1 depicts how the network is set up with the help of the bridge -command. - -![](https://github.com/cloudmesh/cloudmesh-pi-burn/raw/main/images/network-bridge.png) - -Figure 1: Networking Bridge - -**Step 0.** Review and Setup - -At this point, we assume that you have used `cms burn` to create all SD -cards for the Pi's with static IP addresses in the subnet range -`10.1.1.0/24` (excluding `10.1.1.1`. See step 1 for details) - -We are also continuing to use `managerpi` (which is where we burn the -worker SD cards). - -We will now use `cms bridge` to connect the worker Pis to the -internet. Let us again reference the diagram of our network setup. You -should now begin connecting your Pis via the network switch -(unmanaged or managed) if you have not done so already. Ensure that -`managerpi` is also connected to the network switch. - -We assign the eth0 interface of the `managerpi` to be 10.1.1.1, and it -acts as the default gateway for the workers. The workers' IPs were set -during the create command. - -**Step 1.** Configuring our Bridge - -We can easily create our bridge as follows. - -``` -(ENV3) pi@managerpi:~ $ cms bridge create --interface='wlan0' -``` - -We should now reboot. - -``` -(ENV3) pi@managerpi:~ $ sudo reboot -``` - -> Note the `--interface` option indicates the interface used by the -> manager pi to access the internet. In this case, since we are using -> WiFi, it is likely `wlan0`. Other options such as `eth0` and `eth1` -> exist for ethernet connections. - -**Step 2.** Verifying internet connection - -We should now be able to see our workers. - -``` -arp -a -``` - -Note it may take a few minutes for them to populate in the neighbor table. If -you want to speed this up try to ping them individually. - -``` -ping red002 -``` - -At this point, our workers should have internet access. Let us SSH -into one and ping google.com to verify. Ensure you have booted your -workers and connected them to the same network switch as the manager. - -``` -(ENV3) pi@managerpi:~ $ ssh red002 - -pi@red002:~ $ ping google.com -PING google.com (142.250.64.238) 56(84) bytes of data. -64 bytes from mia07s57-in-f14.1e100.net (142.250.64.238): icmp_seq=1 ttl=106 time=48.2 ms -64 bytes from mia07s57-in-f14.1e100.net (142.250.64.238): icmp_seq=2 ttl=106 time=48.3 ms -64 bytes from mia07s57-in-f14.1e100.net (142.250.64.238): icmp_seq=3 ttl=106 time=47.9 ms -64 bytes from mia07s57-in-f14.1e100.net (142.250.64.238): icmp_seq=4 ttl=106 time=47.10 ms -64 bytes from mia07s57-in-f14.1e100.net (142.250.64.238): icmp_seq=5 ttl=106 time=48.5 ms -^C ---- google.com `ping statistics --- -5 packets transmitted, 5 received, 0% packet loss, time 9ms -rtt min/avg/max/mdev = 47.924/48.169/48.511/0.291 ms -``` - -Note how we are able to omit the pi user and .local extension. We have -successfully configured our bridge. Our pis are now ready to cluster. - -## 5. Set up of the SSH keys and SSH tunnel - -One important aspect of a cluster is to setup authentication via ssh -in a convenient way, so we can easily login from the laptop to each of -the PI workers and the PI manager. Furthermore, we like to be able to -login from the PI manager to each of the workers. In addition, we like -to be able to login between the workers. - -We have chosen a very simple setup while relying on ssh tunnel. +## 1. Burning Tutorials -To simplify the setup of this we have developed a command `cms host` -that gathers and scatters keys onto all machines, as well as sets up -the tunnels. +The latest and most up-to-date burning tutorials are hosted on piplanet.org. -It is essential that the key on the laptop must not be password -less. This is also valid for any machine that is directly added to the -network such as in the mesh network. +Please see: +- https://cloudmesh.github.io/pi/tutorial/raspberry-burn/ (burn a raspberry cluster from linux or mac) +- https://cloudmesh.github.io/pi/tutorial/ubuntu-burn/ (burn an ubuntu cluster from linux or mac) +- https://cloudmesh.github.io/pi/tutorial/raspberry-burn-windows/ (burn a raspberry cluster from windows) -To avoid password less keys we recommend you to use `ssh-add` -or `ssh-keychain` which will ask you for one. +## 2. Manual Pages -The manual page for `cms host` is provided in the Manual Pages -section. - -**Step 1.** On the manager create ssh keys for each of the workers. - -``` -(ENV3) pi@managerpi:~ $ cms host key create red00[1-3] -``` - -**Step 2.** On the manager gather the worker, manager, and your laptop -public ssh keys into a file. - -``` -(ENV3) pi@managerpi:~ $ cms host key gather red00[1-3],you@yourlaptop.local ~/.ssh/authorized_keys -``` - -**Step 3.** On the manager scatter the public keys to all the workers -and manager ~/.ssh/authorized_hosts file - -``` -(ENV3) pi@managerpi:~ $ cms host key scatter red00[1-3],localhost ~/.ssh/authorized_keys -``` - -**Step 4.** Remove undeeded keys.txt file - -``` -(ENV3) pi@managerpi:~ $ rm keys.txt -``` - -**Step 5.** Verify SSH reachability from worker to manager and worker to worker. - -``` -(ENV3) pi@managerpi:~ $ ssh red001 -pi@red001:~ $ ssh managerpi.local -``` - -TODO: BUG: The workers still currently still need to use .local after names. This -will be resolved by the inventory create update. - -``` -(ENV3) pi@managerpi:~ $ exit -pi@red001:~ $ ssh red002.local -pi@red002:~ $ exit -pi@red001:~ $ exit -``` - -**Step 6.** (For Bridge setup) Create SSH tunnels on the manager -to enable ssh access from your laptop to the workers - -For now, we manually install autossh, to test the new cms host tunnel -program. Later we add it to the main manager setup script. - -``` -(ENV3) pi@managerpi:~ $ yes y | sudo apt install autossh -(ENV3) pi@managerpi:~ $ cms host tunnel create red00[1-3] -``` - -**Step 7.** (For Bridge setup) Copy the specified command output to -your `~/.ssh/config` file on your laptop - -``` -host tunnel create red00[1-3] - -Using wlan0 IP = 192.168.1.17 -Using cluster hostname = managerpi - -Tunnels created. - -Please place the following in your remote machine's (i.e. laptop) -`~/.ssh/config` file to alias simple ssh access (i.e. `ssh red001`). - -# ---------------------------------------------------------------------- -# copy to ~/.ssh/config on remote host (i.e laptop) -# ---------------------------------------------------------------------- - -Host red001 - HostName managerpi.local - User pi - Port 8001 - -Host red002 - HostName managerpi.local - User pi - Port 8002 - -Host red003 - HostName managerpi.local - User pi - Port 8003 -``` - -> Note: We will in the future provide an addition to the command so you -> can remove and add -> them directly from the command line -> -> ``` -> cms host tunnel config create red00[1-3] -> cms host tunnel config delete red00[1-3] -> ``` - -**Step 8.** (For Bridge setup) Verify SSH reachability from the laptop - to workers - -``` -you@yourlaptop:~ $ ssh red001 -``` - -**Step 9.** Shutdown or reboot your entire cluster using the commands below. - -``` -you@yourlaptop:~ $ cms host shutdown red,red00[1-3] -``` - -``` -you@yourlaptop:~ $ cms host reboot red,red00[1-3] -``` - -## 6. Manual Pages - -### 6.1 Manual Page for the `burn` command +### 2.1 Manual Page for the `burn` command Note to execute the command on the command line you have to type in `cms burn` and not just `burn`. @@ -514,6 +64,18 @@ Note to execute the command on the command line you have to type in [--wifipassword=PSK] [--bs=BLOCKSIZE] [--dryrun] + [--no_diagram] + burn ubuntu NAMES [--inventory=INVENTORY] [--ssid=SSID] [-f] + [--wifipassword=PSK] [-v] --device=DEVICE [--country=COUNTRY] + [--upgrade] + burn raspberry NAMES --device=DEVICE + [--inventory=INVENTORY] + [--ssid=SSID] + [--wifipassword=PSK] + [--country=COUNTRY] + [--password=PASSWORD] + [-v] + [-f] burn firmware check burn firmware update burn install @@ -796,7 +358,6 @@ Examples: ( \ is not shown) > cms burn image delete 2019-09-26-raspbian-buster-lite - ``` @@ -812,7 +373,9 @@ Examples: ( \ is not shown) -### 6.2 Manual Page for the `bridge` command + + +### 2.2 Manual Page for the `bridge` command Note to execute the command on the commandline you have to type in `cms bridge` and not just `bridge`. @@ -842,7 +405,6 @@ Description: bridge create [--interface=INTERFACE] [--ip=IP] [--dns=NAMESERVER] creates the bridge on the current device. A reboot is required. - ``` @@ -850,15 +412,7 @@ Description: - - - - - - - - -### 6.3 Manual Page for the `host` command +### 2.3 Manual Page for the `host` command Note to execute the command on the commandline you have to type in `cms host` and not jsut `host`. @@ -872,7 +426,7 @@ Note to execute the command on the commandline you have to type in host key create NAMES [--user=USER] [--dryrun] [--output=FORMAT] host key list NAMES [--output=FORMAT] host key gather NAMES [--authorized_keys] [FILE] - host key scatter NAMES FILE + host key scatter NAMES FILE [--user=USER] host key add NAMES FILE host key delete NAMES FILE host tunnel create NAMES [--port=PORT] @@ -880,6 +434,12 @@ Note to execute the command on the commandline you have to type in host setup WORKERS [LAPTOP] host shutdown NAMES host reboot NAMES + host adduser NAMES USER + host passwd NAMES USER + host addsudo NAMES USER + host deluser NAMES USER + host config proxy PROXY NAMES [--append] + This command does some useful things. @@ -920,17 +480,21 @@ Description: ssh key gather "red[01-10]" keys.txt - host key scatter HOSTS FILE + host key scatter HOSTS FILE [--user=USER] copies all keys from file FILE to authorized_keys on all hosts, but also makes sure that the users ~/.ssh/id_rsa.pub key is in - the file. + the file. If provided the optional user, it will add the keys to + that user's .ssh directory. This is often required when + adding a new user in which case HOSTS should still a sudo + user with ssh currently enabled. 1) adds ~/.id_rsa.pub to the FILE only if its not already in it 2) removes all duplicated keys Example: ssh key scatter "red[01-10]" + ssh key scatter pi@red[01-10] keys.txt --user=alice host key add NAMES FILE @@ -1016,6 +580,32 @@ Description: Reboots NAMES with `sudo reboot`. If localhost in names, it is rebooted last. + host adduser NAMES USER + + Adds a user with user name USER to the hosts identified by + NAMES. Password is disabled, see host passwd to enable. + + host addsudo NAMES USER + + Adds sudo rights to USER at NAMES + + host passwd NAMES USER + + Changes the password for USER at NAMES + + host deluser NAMES USER + + Deleted USER from NAMES. Home directory will be removed. + + host config proxy PROXY NAMES + + This adds to your ~/.ssh/config file a ProxyJump + configuration to reach NAMES via PROXY. This is useful when + the PROXY is acting as a network bridge for NAMES to your + current device. + + Example: + cms host config proxy pi@red.lcaol red00[1-2] ``` @@ -1031,7 +621,9 @@ Description: -### 6.4 Manual Page for the `pi` command + + +### 2.4 Manual Page for the `pi` command Note to execute the command on the command line you have to type in `cms pi` and not just `pi`. @@ -1139,14 +731,15 @@ Description: pi script list SERVICE NAMES pi script list - ``` -### 6.4 Manual Page for the `ssh` command + + +### 2.5 Manual Page for the `ssh` command Note to execute the command on the command line you have to type in `cms ssh` and not just `ssh`. @@ -1156,13 +749,9 @@ file via the commandline ``` - ssh ssh config list [--output=OUTPUT] ssh config add NAME IP [USER] [KEY] ssh config delete NAME - ssh host delete NAME - ssh host add NAME - ssh [--name=VMs] [--user=USERs] [COMMAND] Arguments: NAME Name or ip of the machine to log in @@ -1172,51 +761,38 @@ Arguments: parameters to the ssh config file. if the resource exists, it will be overwritten. The information will be written in /.ssh/config + USER The username for the ssh resource + KEY The location of the public keye used for + authentication to the host Options: - -v verbose mode --output=OUTPUT the format in which this list is given formats includes cat, table, json, yaml, dict. If cat is used, it is just printed as is. [default: table] - --user=USERs overwrites the username that is - specified in ~/.ssh/config - --name=CMs the names of the VMS to execute the - command on Description: ssh config list - lists the hostsnames that are present in the - ~/.ssh/config file + lists the hostsnames that are present in the ~/.ssh/config file ssh config add NAME IP [USER] [KEY] registers a host i ~/.ssh/config file Parameters are attribute=value pairs - Note: Note yet implemented - ssh [--name=VMs] [--user=USERs] [COMMAND] - executes the command on the named hosts. If user is - specified and is greater than 1, it must be specified for - each vm. If only one username is specified it is used for - all vms. However, as the user is typically specified in the - cloudmesh database, you probably do not have to specify - it as it is automatically found. + ssh config delete NAME + deletes the named host from the ssh config file Examples: - - ssh config add blue 192.168.1.245 blue + ssh config add blue 192.168.1.245 gregor Adds the following to the !/.ssh/config file Host blue HostName 192.168.1.245 - User blue + User gergor IdentityFile ~/.ssh/id_rsa.pub - - - ``` @@ -1226,17 +802,21 @@ Examples: -## 7. FAQ and Hints + + +## 3. FAQ and Hints Here, we provide some useful FAQs and hints. -### 7.1 Quickstart for a Setup of a cluster from macOS or Linux with no burning on a PI. +> Note: many of these are out of date or no longer required, but are kept for posterity. + +### 3.1 Quickstart for a Setup of a cluster from macOS or Linux with no burning on a PI. This will setup the same cluster seen in [Quickstart for Bridged WiFi](#quickstart-for-bridged-wifi). Pi imager and a manual manager pi setup is not required using this method. It will use the latest Pi OS images, full for master, and lite for workers. -#### 7.1.1 Prerequisites +#### 3.1.1 Prerequisites * We recommend Python 3.8.2 Python or newer. * We recommend pip version 21.0.0 or newer @@ -1244,7 +824,7 @@ images, full for master, and lite for workers. ssh/id_rsa.pub * macOS dependencies [What packages do I need to run the info command on macOS](#what-packages-do-i-need-to-run-the-info-command-on-macos) -#### 7.1.2 Install Cloudmesh +#### 3.1.2 Install Cloudmesh Create a Python virtual environment `ENV3` in which to install cloudmesh. This will keep cloudmesh and its dependencies separate from your default @@ -1261,7 +841,7 @@ you@laptop:~ $ source ~/ENV3/bin/activate (ENV3) you@laptop:~/cm $ cloudmesh-installer get pi ``` -#### 7.1.3 Create a Cluster +#### 3.1.3 Create a Cluster Here, we demonstarte how to burn 1 manager and 2 worker SD Cards. The manager is called red, the workers are red001 and red002. @@ -1360,7 +940,7 @@ pi temp red,red00[1-2] -### 7.2 Quickstart for Using a Pi to Burn a Cluster Using Inventory +### 3.2 Quickstart for Using a Pi to Burn a Cluster Using Inventory In this guide, we will show how you can configure a Cloudmesh Inventory to easily burn a cluster of SD cards as well as configure the current Pi as the @@ -1371,7 +951,7 @@ previous section (see Figure 1). The requirements for this guide are the same as the [Quickstart for Bridged WiFi](#quickstart-for-bridged-wifi). -#### 7.2.1 Initial Manager Setup +#### 3.2.1 Initial Manager Setup Ensure you have burned an SD card from your laptop using [Raspberry Pi Imager](#https://www.raspberrypi.org/software/). Ensure you burn the card with @@ -1402,7 +982,7 @@ Reboot after this script pi@managerpi:~ $ sudo reboot ``` -#### 7.2.2 Creating our inventory +#### 3.2.2 Creating our inventory For this guide, we will create two workers for `managerpi`. We can do this as follows: @@ -1425,7 +1005,7 @@ inventory list --inventory=cluster.yaml +-----------+-----------+------+-------------+---------+-------+---------+----------+----------+-----+---------+--------+---------+-------------+-------------------+----------+ ``` -#### 7.2.3 Burning SD Cards using Inventory +#### 3.2.3 Burning SD Cards using Inventory First, verify that you have plugged in your SD card writer with an SD card into the `managerpi`. For this guide, we will simply use one SD card burner to burn @@ -1487,7 +1067,7 @@ We must now reboot the manager. (ENV3) pi@managerpi:~ $ sudo reboot ``` -#### 7.2.4 Booting Up Workers and Verifying Connection +#### 3.2.4 Booting Up Workers and Verifying Connection Insert the burned worker cards into the worker Pis and boot. @@ -1503,7 +1083,7 @@ pi temp red002 +--------+--------+-------+----------------------------+ ``` -### 7.3 Can I use the LEDs on the PI Motherboard? +### 3.3 Can I use the LEDs on the PI Motherboard? Typically this LED is used to communicate some system-related information. However `cms pi` can control it to switch status on @@ -1519,7 +1099,7 @@ installed you switch the red LED off. For more options see the manual page -### 7.4 How can I use pycharm, to edit files or access files in general from my Laptop on the PI? +### 3.4 How can I use pycharm, to edit files or access files in general from my Laptop on the PI? This is easily possible with the help of SSHFS. To install it we refer you to See also: SSHFS: add @@ -1558,7 +1138,7 @@ umount redcm If you need other directories, pleas apply our strategy accordingly -### 7.5 How can I enhance the `get` script? +### 3.5 How can I enhance the `get` script? Instead of using the link @@ -1575,7 +1155,7 @@ You can create a pull request at * -### 7.6 Can I use a Mesh Network for the setup? +### 3.6 Can I use a Mesh Network for the setup? This section is still under development. @@ -1591,7 +1171,7 @@ Figure 2: Networking with Mesh network You will not need the bridge command to setup the network. -### 7.7 Can I use cms burn on Linux? +### 3.7 Can I use cms burn on Linux? Not everything is supported. @@ -1629,7 +1209,7 @@ For the full features, please use `cms burn create` instead of `cms burn sdcard` -### 7.8 What packages do I need to run the info command on macOS +### 3.8 What packages do I need to run the info command on macOS ``` brew install libusb @@ -1645,7 +1225,7 @@ which does cost $40 for a license. For this reason, we recommend that you first set up the manager PI and do all burning on the manager PI. -### 7.9 Are there any unit tests? +### 3.9 Are there any unit tests? As `cms burn` may delete and format files and disks/SD Cards during unit testing users are supposed to first review the tests before running @@ -1661,7 +1241,7 @@ We have the following tests: * TODO: add the other tests -### 7.10 Using Pi Imager to setup a Manager Pi with headless access +### 3.10 Using Pi Imager to setup a Manager Pi with headless access This FAQ will provide step-by-step instructions for burning and accessing a headless manager pi. We include instructions for either wifi access to @@ -1817,7 +1397,7 @@ cms burn automates this process for you. pi@managerpi:~ $ ``` -### 7.11 Single Card Burning +### 3.11 Single Card Burning Step 0. Ensure the SD card is inserted. @@ -1847,7 +1427,7 @@ to remove the SD card. We can now proceed to [the bridge setup](#connecting-pis-to-the-internet-via-bridge ) -### 7.12 How to update firmware? +### 3.12 How to update firmware? To update the firmware reference the [raspi documentation](#https://www.raspberrypi.org/documentation/hardware/raspberrypi/booteeprom.md) @@ -1861,7 +1441,7 @@ pi@managerpi:~ $ sudo rpi-eeprom-update -a pi@managerpi:~ $ sudo reboot ``` -### 7.13 Alternatives +### 3.13 Alternatives There are several alternatives to make the setup easier: @@ -1876,7 +1456,7 @@ There are several alternatives to make the setup easier: clusters this requires multiple Servers so that the network is not overwhelmed. Starting the cluster takes much longer. -### 7.14 How do I scann for WIFI networks? +### 3.14 How do I scann for WIFI networks? ``` sudo iwlist wlan0 scan @@ -1921,7 +1501,8 @@ sudo iwlist wlan0 scan * TODO1 = todo for boot fs, rootfs not supported -### 7.16 I run into a Kernal Panic on my burned Pi. What do I do? +### 3.16 I run into a Kernal Panic on my burned Pi. What do I do? + Occassionally, one may run into an error similar to the following: ``` @@ -1930,15 +1511,23 @@ Kernel panic-not syncing: VFS: unable to mount root fs on unknown-block(179,2) See [here](https://raspberrypi.stackexchange.com/questions/40854/kernel-panic-not-syncing-vfs-unable-to-mount-root-fs-on-unknown-block179-6) for more information on this bug. -This error has been reported in the past. A simple reburn using `cms burn` tends to resolve the issue. +This error has been reported in the past. A simple reburn using `cms burn` +tends to resolve the issue. + +### 3.17 How do I enable password login? + +The option `--set_passwd` in `cms burn cluster` enables you to securely enter a +password to prevent the password disable. -### 7.17 How do I enable password login? +The option `[--passwd=PASSWD]` is used with `cms burn create` todo the same +thing. Note entering the passwd in the command is optional.If empty you will be +prompted. -The option `--set_passwd` in `cms burn cluster` enables you to securely enter a password to prevent the password disable. +### 3.18 How do I use SDCard externers with different voltage? -The option `[--passwd=PASSWD]` is used with `cms burn create` todo the same thing. Note entering the passwd in the command is optional.If empty you will be prompted. -### 7.18 Becuase I am using and sd card extender, I need to set a cmdline argument to force 3.3V SD card operation. +Becauase I am using and sd card extender, I need to set a cmdline argument to +force 3.3V SD card operation. You can set an arbitray command line argument with @@ -1952,7 +1541,34 @@ To force 3.3V operation to enable the use of an SD card extender use cms burn set--cmdline=sdhci.quirks2=4 ``` -## 8. How can I contribute Contributing +### 3.19 How do I get the latest image if a new image was released? + +From time to time raspberry.org releases new operating systems. To assure you +get the latest version, you can do the following to download the latest lite +abd full images : + +```bash +$ cms burn image versions --refresh +$ cms burn image get latest-lite +$ cms burn image get latest-full +``` + +To safe space you can also delete the old versions. Look at the storage +location where we place the images with + +```bash +$ ls -1 ~/.cloudmesh/cmburn/images +``` + +YOu can delete the ones that do not have the lates date. Such as + +```bash +$ rm ~/.cloudmesh/cmburn/images/2021-01-11-raspio* +``` + +If you see any images with the date 2021-01-11 and so on. + +## 4. How can I contribute Contributing The code uses a variety of cloudmesh components. This mainly includes diff --git a/VERSION b/VERSION index 9e7bcc83..af129e63 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -4.3.19 +4.3.29 diff --git a/cloudmesh/burn/Iso.py b/cloudmesh/burn/Iso.py index 70208c93..b31d6ba0 100644 --- a/cloudmesh/burn/Iso.py +++ b/cloudmesh/burn/Iso.py @@ -1,6 +1,8 @@ import os +# import wget from cloudmesh.burn.image import Image +from cloudmesh.common.Shell import Shell class Iso: @@ -13,4 +15,6 @@ def get(tag="latest"): url = Iso.distribution[tag] destination = Image().directory + os.path.basename(url) - os.system(f'wget -O {destination} {url}') + Shell.download(url, destination, provider='system') + # wget.download(url, out=destination) + # os.system(f'wget -O {destination} {url}') diff --git a/cloudmesh/burn/__init__.py b/cloudmesh/burn/__init__.py index 17643650..ec23a6c0 100644 --- a/cloudmesh/burn/__init__.py +++ b/cloudmesh/burn/__init__.py @@ -1 +1 @@ -__version__ = "4.3.19" +__version__ = "4.3.29" diff --git a/cloudmesh/burn/__version__.py b/cloudmesh/burn/__version__.py index 7a26ffe5..e37d271d 100644 --- a/cloudmesh/burn/__version__.py +++ b/cloudmesh/burn/__version__.py @@ -1 +1 @@ -version = "4.3.19" +version = "4.3.29" diff --git a/cloudmesh/burn/burner/Burner.py b/cloudmesh/burn/burner/Burner.py index c9d0c43a..760e4c74 100644 --- a/cloudmesh/burn/burner/Burner.py +++ b/cloudmesh/burn/burner/Burner.py @@ -1,4 +1,5 @@ import os +# import wget from cloudmesh.burn.burner.raspberryos import Burner as RaspberryOsBurner from cloudmesh.burn.usb import USB @@ -53,11 +54,12 @@ def install(self): return "" else: banner("Installing pishrink.sh into /usr/local/bin") + script_name = Shell.download( + 'https://raw.githubusercontent.com/Drewsif/PiShrink/master/pishrink.sh', 'pishrink.sh', provider='system') script = \ - """ - wget https://raw.githubusercontent.com/Drewsif/PiShrink/master/pishrink.sh - chmod +x pishrink.sh - sudo mv pishrink.sh /usr/local/bin + f""" + chmod +x {script_name} + sudo mv {script_name} /usr/local/bin """ result = JobScript.execute(script) diff --git a/cloudmesh/burn/burner/RaspberryBurner.py b/cloudmesh/burn/burner/RaspberryBurner.py index 384b8e86..2cbc36b2 100644 --- a/cloudmesh/burn/burner/RaspberryBurner.py +++ b/cloudmesh/burn/burner/RaspberryBurner.py @@ -1,13 +1,22 @@ +import os import time +from getpass import getpass from cloudmesh.burn.burner.BurnerABC import AbstractBurner +from cloudmesh.burn.image import Image from cloudmesh.burn.raspberryos.cmdline import Cmdline from cloudmesh.burn.raspberryos.runfirst import Runfirst from cloudmesh.burn.sdcard import SDCard from cloudmesh.burn.usb import USB +from cloudmesh.burn.wifi.ssid import get_ssid +from cloudmesh.common.Host import Host +from cloudmesh.common.Shell import Shell from cloudmesh.common.console import Console from cloudmesh.common.parameter import Parameter -from cloudmesh.common.util import yn_choice, readfile +from cloudmesh.common.util import banner +from cloudmesh.common.util import path_expand +from cloudmesh.common.util import readfile +from cloudmesh.common.util import yn_choice from cloudmesh.inventory.inventory import Inventory @@ -17,21 +26,78 @@ class Burner(AbstractBurner): Inventory should contain information on manager and workers """ - def __init__(self, inventory=None): + def __init__(self, inventory=None, names=None, ssid=None, + wifipassword=None, force_inv=False, country=None): # Get inventory + self.ssid = ssid + self.wifipasswd = wifipassword if inventory is None: - inv = Inventory() + names = Parameter.expand(names) + manager, workers = Host.get_hostnames(names) + if workers: + worker_base_name = ''.join( + [i for i in workers[0] if not i.isdigit()]) + + cluster_name = manager or worker_base_name + inventory = path_expand(f'~/.cloudmesh/inventory-{cluster_name}.yaml') + + if not os.path.exists(inventory) or force_inv: + if not manager: + Console.error("No inventory found. Can not create an " + "inventory without a " + "manager.") + return "" + + Inventory.build_default_inventory(filename=inventory, + manager=manager, + workers=workers) + inv = Inventory(filename=inventory) + else: inv = Inventory(filename=inventory) + self.inventory = inv # Find managers and workers managers = inv.find(service='manager') + if len(managers) > 0: + if not self.ssid: + self.ssid = get_ssid() + if self.ssid == "": + Console.info('Could not determine SSID, skipping wifi ' + 'config') + self.ssid = None + if not self.wifipasswd and self.ssid: + self.wifipasswd = getpass(f"Using --SSID={self.ssid}, please " + f"enter wifi password:") workers = inv.find(service='worker') - # No inherenet need to distinguish the configs by service configs = managers + workers # Create dict for them for easy lookup self.configs = dict((config['host'], config) for config in configs) + self.get_images() + self.country = country if country else Shell.locale().upper() + + def get_images(self): + """ + Downloads all tags found in self.configs + """ + tags = set() + for config in self.configs.values(): + try: + tags.add(config['tag']) + except KeyError as e: + Console.warning(f'Could not find tag for {config["host"]}. Skipping') + + banner("Downloading Images", figlet=True) + + image = Image() + + for tag in tags: + Console.info(f'Attempting to download {tag}') + res = image.fetch(tag=[tag]) + if not res: + Console.error('Failed Image Fetch.') + raise Exception('Failed Image Fetch') def cluster(self, arguments=None): raise NotImplementedError @@ -41,9 +107,7 @@ def burn(self, device=None, verbose=False, password=None, - ssid=None, - wifipasswd=None, - country=None): + ): """ Given the name of a config, burn device with RaspberryOS and configure properly """ @@ -68,6 +132,8 @@ def burn(self, print() return "" + banner(f"Burn {name}", figlet=True) + # Confirm card is inserted into device path if not yn_choice(f'Is the card to be burned for {name} inserted?'): if not yn_choice(f"Please insert the card to be burned for {name}. " @@ -103,8 +169,8 @@ def burn(self, runfirst.set_password(password=password) runfirst.set_locale(timezone=config['timezone'], locale=config['locale']) - if ssid: - runfirst.set_wifi(ssid, wifipasswd, country=country) + if self.ssid and 'wifi' in config['services']: + runfirst.set_wifi(self.ssid, self.wifipasswd, self.country) runfirst.set_key(key=readfile(config['keyfile']).strip()) if 'bridge' in config['services']: @@ -120,16 +186,14 @@ def burn(self, return def inventory(self, arguments=None): - raise NotImplementedError + return self.inventory def multi_burn(self, names=None, devices=None, verbose=False, password=None, - ssid=None, - wifipasswd=None, - country=None): + ): """ Given multiple names, burn them """ @@ -152,10 +216,7 @@ def multi_burn(self, name=name, device=devices[0], verbose=verbose, - password=password, - ssid=ssid, - wifipasswd=wifipasswd, - country=None + password=password ) Console.ok('Finished burning all cards') diff --git a/cloudmesh/burn/command/burn.py b/cloudmesh/burn/command/burn.py index e98a0b90..a34565e4 100644 --- a/cloudmesh/burn/command/burn.py +++ b/cloudmesh/burn/command/burn.py @@ -49,7 +49,7 @@ def do_burn(self, args, arguments): [--bs=BLOCKSIZE] [--dryrun] [--no_diagram] - burn ubuntu NAMES [--inventory=INVENTORY] [--ssid=SSID] + burn ubuntu NAMES [--inventory=INVENTORY] [--ssid=SSID] [-f] [--wifipassword=PSK] [-v] --device=DEVICE [--country=COUNTRY] [--upgrade] burn raspberry NAMES --device=DEVICE @@ -60,7 +60,6 @@ def do_burn(self, args, arguments): [--password=PASSWORD] [-v] [-f] - [--timezone=TIMEZONE] burn firmware check burn firmware update burn install @@ -426,53 +425,39 @@ def execute(label, function): return "" elif arguments.raspberry: - banner(txt="RaspberryOS Burn", figlet=True) - names = Parameter.expand(arguments.NAMES) - manager, workers = Host.get_hostnames(names) - ssid = arguments['--ssid'] - wifipasswd = arguments['--wifipassword'] + banner(txt="RaspberryOS Burn", figlet=True) if arguments.inventory: - burner = RaspberryBurner(inventory=arguments.inventory) + inv_path = path_expand(f'~/.cloudmesh/{arguments.inventory}') + try: + burner = RaspberryBurner(inventory=inv_path, + ssid=arguments['--ssid'], + wifipassword=arguments['--wifipassword'], + country=arguments['--country']) + except: + Console.error('Burner Error') + return "" else: - if workers: - worker_base_name = ''.join( - [i for i in workers[0] if not i.isdigit()]) + try: + burner = RaspberryBurner( + names=arguments.NAMES, + ssid=arguments['--ssid'], + wifipassword=arguments['--wifipassword'], + force_inv=arguments['-f'], + country=arguments['--country'] + ) + except Exception as e: + Console.error('Burner Error') + raise e - cluster_name = manager or worker_base_name - inventory = path_expand(f'~/.cloudmesh/inventory-{cluster_name}.yml') - - if not os.path.exists(inventory) or arguments['-f']: - if not manager: - Console.error("No inventory found. Can not create an " - "inventory without a " - "manager.") - return "" - - _build_default_inventory(filename=inventory, - manager=manager, workers=workers) - - burner = RaspberryBurner(inventory=inventory) - - if manager: - if not ssid: - ssid = get_ssid() - if ssid == "": - Console.info('Could not determine SSID, skipping wifi ' - 'config') - if not wifipasswd and not ssid == "": - wifipasswd = getpass(f"Using --SSID={ssid}, please " - f"enter wifi password:") + return "" execute("burn raspberry", burner.multi_burn( names=arguments.NAMES, devices=arguments.device, verbose=arguments['-v'], password=arguments['--password'], - ssid=ssid, - wifipasswd=wifipasswd, - country=arguments['--country'] )) return "" @@ -487,8 +472,47 @@ def execute(label, function): c = Configure(inventory=arguments.inventory, debug=arguments['-v']) inv = Inventory(filename=arguments.inventory) else: - c = Configure(debug=arguments['-v']) - inv = Inventory() + names = Parameter.expand(arguments.NAMES) + manager, workers = Host.get_hostnames(names) + if workers: + worker_base_name = ''.join( + [i for i in workers[0] if not i.isdigit()]) + + cluster_name = manager or worker_base_name + inventory = path_expand(f'~/.cloudmesh/inventory-{cluster_name}.yaml') + + if not os.path.exists(inventory) or arguments['-f']: + if not manager: + Console.error("No inventory found. Can not create an " + "inventory without a " + "manager.") + return "" + + Inventory.build_default_inventory(filename=inventory, + manager=manager, + workers=workers, + manager_image='ubuntu-20.10-64-bit', + worker_image='ubuntu-20.10-64-bit') + + c = Configure(inventory=inventory, debug=arguments['-v'], download_images=True) + inv = Inventory(filename=inventory) + + names = Parameter.expand(arguments.NAMES) + manager, workers = Host.get_hostnames(names) + if manager: + if not arguments.ssid and 'wifi' in c.configs[manager]['services']: + arguments.ssid = get_ssid() + if arguments.ssid == "": + Console.info('Could not determine SSID, skipping wifi ' + 'config') + arguments.ssid = None + if not arguments.wifipassword and arguments.ssid is not None: + arguments.country = Shell.locale().upper() + arguments.wifipassword = getpass(f"Using --SSID=" + f"{arguments.ssid} and " + f" --COUNTRY=" + f"{arguments.country}, please " + f"enter wifi password:") # Probably not the best way to get the tag, but we assume all tags # are the same for each row for now @@ -513,19 +537,19 @@ def execute(label, function): # determine if we are burning a manager, as this needs to be done # first to get the ssh public key - manager = False - for name in names: - if not inv.has_host(name): - Console.error(f'Could not find {name} in inventory {inv.filename}') - return "" - service = inv.get(name=name, attribute='service') - if service == 'manager' and not manager: - manager = name - # make manager first in names - names.remove(name) - names.insert(0, name) - elif service == 'manager' and manager: - raise Exception('More than one manager detected in NAMES') + # manager = False + # for name in names: + # if not inv.has_host(name): + # Console.error(f'Could not find {name} in inventory {inv.filename}') + # return "" + # service = inv.get(name=name, attribute='service') + # if service == 'manager' and not manager: + # manager = name + # # make manager first in names + # names.remove(name) + # names.insert(0, name) + # elif service == 'manager' and manager: + # raise Exception('More than one manager detected in NAMES') for name in names: if not yn_choice(f'Is the card to be burned for {name} inserted?'): @@ -552,10 +576,10 @@ def execute(label, function): sdcard.mount(device=arguments.device, card_os="ubuntu") if service == 'manager': # Generate a private public key pair for the manager that will be persistently used - priv_key, pub_key = c.generate_ssh_key(name) + # priv_key, pub_key = c.generate_ssh_key(name) # Write priv_key and pub_key to /boot/id_rsa and /boot/id_rsa.pub - SDCard.writefile(filename=f'{sdcard.boot_volume}/id_rsa', content=priv_key) - SDCard.writefile(filename=f'{sdcard.boot_volume}/id_rsa.pub', content=pub_key) + # SDCard.writefile(filename=f'{sdcard.boot_volume}/id_rsa', content=priv_key) + # SDCard.writefile(filename=f'{sdcard.boot_volume}/id_rsa.pub', content=pub_key) c.build_user_data(name=name, country=arguments.country, upgrade=arguments.upgrade, @@ -1001,57 +1025,3 @@ def execute(label, function): Console.error("see manual page: cms help burn") return "" - - -def _build_default_inventory(filename, manager, workers, ips=None, images=None): - # cms inventory add red --service=manager --ip=10.1.1.1 --tag=latest-lite - # --timezone="America/Indiana/Indianapolis" --locale="us" - # cms inventory set red services to "bridge" --listvalue - # cms inventory add "red0[1-3]" --service=worker --ip="10.1.1.[2-4]" - # --router=10.1.1.1 --tag=latest-lite --timezone="America/Indiana/Indianapolis" --locale="us" - # cms inventory set "red0[1-3]" dns to "8.8.8.8,8.8.4.4" --listvalue - - Console.info("No inventory found or forced rebuild. Buidling inventory " - "with defaults.") - Shell.execute("rm", arguments=[ - '-f', filename]) - i = Inventory(filename=filename) - timezone = Shell.timezone() - locale = Shell.locale() - manager_ip = ips[0] if ips else '10.1.1.1' - image = images[0] if images else 'latest-lite' - element = {} - element['host'] = manager - element['status'] = 'inactive' - element['service'] = 'manager' - element['ip'] = manager_ip - element['tag'] = image - element['timezone'] = timezone - element['locale'] = locale - element['services'] = ['bridge', 'wifi'] - element['keyfile'] = '~/.ssh/id_rsa.pub' - i.add(**element) - i.save() - - last_octet = 2 - index = 1 - for worker in workers: - ip = ips[index] if ips else f'10.1.1.{last_octet}' - image = images[index] if images else 'latest-lite' - element = {} - element['host'] = worker - element['status'] = 'inactive' - element['service'] = 'worker' - element['ip'] = ip - element['tag'] = image - element['timezone'] = timezone - element['locale'] = locale - element['router'] = manager_ip - element['dns'] = ['8.8.8.8', '8.8.4.4'] - element['keyfile'] = '~/.ssh/id_rsa.pub' - i.add(**element) - i.save() - last_octet += 1 - index += 1 - - print(i.list(format="table")) diff --git a/cloudmesh/burn/gui.py b/cloudmesh/burn/gui.py index b3cdae90..44299698 100644 --- a/cloudmesh/burn/gui.py +++ b/cloudmesh/burn/gui.py @@ -17,7 +17,7 @@ from cloudmesh.common.util import path_expand from cloudmesh.diagram.diagram import Diagram from cloudmesh.burn.wifi.ssid import get_ssid -from cloudmesh.burn.command.burn import _build_default_inventory +from cloudmesh.inventory.inventory import Inventory def _execute(command): @@ -46,13 +46,18 @@ def image(name): }, "os_ubuntu_64bit_20_04": { "name": "Ubuntu 64-bit 20.04", - "manager": "Ubuntu 64 04", - "worker": "Ubuntu 64 04" + "manager": "ubuntu-20.04.2-64-bit ", + "worker": "ubuntu-20.04.2-64-bit " }, "os_ubuntu_64bit_20_10": { "name": "Ubuntu 64-bit 20.10", - "manager": "Ubuntu 64 10", - "worker": "Ubuntu 64 10" + "manager": "ubuntu-20.10-64-bit", + "worker": "ubuntu-20.10-64-bit" + }, + "os_ubuntu_64bit_20_10_desktop": { + "name": "Ubuntu 64-bit 20.10 desktop", + "manager": "ubuntu-desktop", + "worker": "ubuntu-desktop" } } @@ -219,7 +224,7 @@ def line(msg): default = count == 0 if os_is_linux(): burn_layout.append( - [sg.Radio(device, group_id="DEVICE", + [sg.Radio(device['dev'], group_id="DEVICE", default=default, key=f"device-{device['name']}")] ) @@ -466,6 +471,7 @@ def run(self): device = "/dev/" + entry.replace("device-", "") if str(entry).startswith("tag") and values[entry]: tags.append(values[entry]) + key = values['key'] self.hostnames_str = ','.join(hostnames) self.ips_str = ','.join(ips) @@ -514,16 +520,24 @@ def run(self): # f" {self.imaged_str}" manager, workers = Host.get_hostnames(hostnames) - filename = path_expand(f"~/.cloudmesh/inventory-{manager}.yml") - _build_default_inventory(filename=filename, manager=manager, - workers=workers, ips=ips, images=tags) + filename = path_expand(f"~/.cloudmesh/inventory-{manager}.yaml") + Inventory.build_default_inventory(filename=filename, manager=manager, + workers=workers, ips=ips, + gui_images=tags) + + if "ubuntu" in tags[0]: + os_cmd = 'ubuntu' + else: + os_cmd = 'raspberry' + if host == manager: - command = f"cms burn raspberry {host}" \ + command = f"cms burn {os_cmd} {host}" \ f" --device={device}" \ f" --ssid={self.ssid}" \ - f" --wifipassword={self.wifipassword}" + f" --wifipassword={self.wifipassword}" \ + f" --country={Shell.locale().upper()}" else: - command = f"cms burn raspberry {host}" \ + command = f"cms burn {os_cmd} {host}" \ f" --device={device}" print(command) diff --git a/cloudmesh/burn/hardware.py b/cloudmesh/burn/hardware.py index 6d74f528..6391c99d 100644 --- a/cloudmesh/burn/hardware.py +++ b/cloudmesh/burn/hardware.py @@ -1,5 +1,6 @@ import os import socket +import platform class Hardware(object): @@ -12,7 +13,10 @@ def is_pi(): :return: returns true if this is called on a Pi :rtype: bool """ - return os.uname()[4][:3] == 'arm' and 'Raspberry' in Hardware.model() + try: + return os.uname()[4][:3] == 'arm' and 'Raspberry' in Hardware.model() + except: + return False @staticmethod def get_mac(interface='eth0'): diff --git a/cloudmesh/burn/image.py b/cloudmesh/burn/image.py index 51a46cb0..06258e34 100644 --- a/cloudmesh/burn/image.py +++ b/cloudmesh/burn/image.py @@ -1,5 +1,6 @@ import os import textwrap +# import wget import zipfile from pathlib import Path @@ -10,6 +11,7 @@ from cloudmesh.burn.util import sha256sum from cloudmesh.common.Tabulate import Printer from cloudmesh.common.console import Console +from cloudmesh.common.Shell import Shell from cloudmesh.common.util import banner from cloudmesh.common.util import path_expand from cloudmesh.common.util import readfile, writefile @@ -112,7 +114,11 @@ def __init__(self): self.raspberry_images = { "lite": "https://downloads.raspberrypi.org/raspios_lite_armhf/images", - "full": "https://downloads.raspberrypi.org/raspios_full_armhf/images" + "full": "https://downloads.raspberrypi.org/raspios_full_armhf/images", + "lite-64": "https://downloads.raspberrypi.org/raspios_lite_arm64/images/", + "full-64": "https://downloads.raspberrypi.org/raspios_arm64/images/", + "lite-legacy": "https://downloads.raspberrypi.org/raspios_oldstable_lite_armhf/images", + "full-legacy": "https://downloads.raspberrypi.org/raspios_oldstable_armhf/images/" } # if name == 'latest': # self.fullpath = self.directory + '/' + self.latest_version() + '.img' @@ -140,12 +146,14 @@ def find(tag=None): :rtype: dict """ tag = tag or ['latest-lite'] + if not isinstance(tag,list): + tag = [tag] found = [] data = Image.create_version_cache(refresh=False) for entry in data: match = True for t in tag: - match = match and t in entry["tag"] + match = match and t == entry["tag"] if match: found.append(entry) if len(found) == 0: @@ -158,7 +166,7 @@ def create_version_cache(refresh=False): """ creates a cache of all released pi images - :param refresh: reresd it from the Web if True + :param refresh: refresh it from the Web if True :type refresh: bool :return: writes it into ~/.cloudmesh/cmburn/distributions.yaml :rtype: file @@ -166,7 +174,11 @@ def create_version_cache(refresh=False): data = { "lite": [], - "full": [] + "full": [], + "lite-64": [], + "full-64": [], + "lite-legacy": [], + "full-legacy": [] } cache = Path(os.path.expanduser("~/.cloudmesh/cmburn/distributions.yaml")) @@ -201,12 +213,17 @@ def fetch_kind(kind=None): os.system("mkdir -p ~/.cloudmesh/cmburn") fetch_kind(kind="lite") fetch_kind(kind="full") + fetch_kind(kind="lite-64") + fetch_kind(kind="full-64") + fetch_kind(kind="lite-legacy") + fetch_kind(kind="full-legacy") writefile(cache, yaml.dump(data)) data = readfile(cache) data = yaml.safe_load(readfile(cache)) # convert to array - result = data["lite"] + data["full"] + Ubuntu.distribution + result = data["lite"] + data["full"] + data["lite-64"] + data["full-64"] \ + + data["lite-legacy"] + data["full-legacy"]+ Ubuntu.distribution return result @@ -235,7 +252,7 @@ def find_image_zip(repo=None, version=None): result = requests.get(url, verify=False) lines = result.text.split(' ') for line in lines: - if '.zip"' in line and "" in line: + if ('.zip"' in line) or (".xz" in line) and "" in line: line = line.split('href="')[1] line = line.split('"')[0] link = f"{repo}/{version}/{line}" @@ -252,7 +269,7 @@ def latest_version(kind="lite"): @staticmethod def get_name(url): - return os.path.basename(url).replace('.zip', '') + return os.path.basename(url).replace('.zip', '').replace('.xz', '') # noinspection PyBroadException def fetch(self, url=None, tag=None, verify=True): @@ -307,7 +324,9 @@ def fetch(self, url=None, tag=None, verify=True): Console.warning(f"The file is already downloaded. Found at:\n\n" f" {img_file}\n") return img_file - os.system(f'wget -O {xz_filename} {image["url"]}') + Shell.download(image["url"], xz_filename, provider='system') + # wget.download(image["url"], out=xz_filename) + # os.system(f'wget -O {xz_filename} {image["url"]}') print(f"Extracting {img_filename}") self.unzip_image(xz_filename) @@ -322,6 +341,8 @@ def fetch(self, url=None, tag=None, verify=True): size = requests.get(image["url"], verify=False, stream=True).headers['Content-length'] zip_filename = os.path.basename(source_url) img_filename = zip_filename.replace('.zip', '.img') + #NEW + img_filename = zip_filename.replace('.img.xz', '.img') sha1_filename = zip_filename + '.sha1' sha256_filename = zip_filename + '.sha256' @@ -343,10 +364,16 @@ def fetch(self, url=None, tag=None, verify=True): image['sha1'] = image['url'] + ".sha1" image['sha256'] = image['url'] + ".sha256" if verify: - os.system(f'wget -O {sha1_filename} {image["sha1"]}') - os.system(f'wget -O {sha256_filename} {image["sha256"]}') + Shell.download(image["sha1"], sha1_filename, provider='system') + Shell.download(image["sha256"], sha256_filename, provider='system') + # wget.download(image["sha1"], out=sha1_filename) + # wget.download(image["sha256"], out=sha256_filename) + # os.system(f'wget -O {sha1_filename} {image["sha1"]}') + # os.system(f'wget -O {sha256_filename} {image["sha256"]}') - os.system(f'wget -O {zip_filename} {image["url"]}') + Shell.download(image["url"], zip_filename, provider='system') + # wget.download(image["url"], out=zip_filename) + # os.system(f'wget -O {zip_filename} {image["url"]}') if verify: sha1 = sha1sum(zip_file) @@ -371,7 +398,10 @@ def fetch(self, url=None, tag=None, verify=True): print(f"Extracting {img_filename}") self.unzip_image(zip_filename) - Path(zip_filename).unlink() + try: + Path(zip_filename).unlink() + except: + pass return img_filename def unzip_image(self, zip_filename=None): @@ -407,7 +437,7 @@ def rm(self, image="lite"): # if tag == "latest": # tag = latest_version(kind="lite") - for ending in [".img", ".zip"]: + for ending in [".img", ".zip", ".xz"]: try: Path(Path(self.directory) / Path(image + ending)).unlink() except Exception as e: # noqa: F841 diff --git a/cloudmesh/burn/raspberryos/runfirst.py b/cloudmesh/burn/raspberryos/runfirst.py index 239680f2..eae5e603 100644 --- a/cloudmesh/burn/raspberryos/runfirst.py +++ b/cloudmesh/burn/raspberryos/runfirst.py @@ -123,6 +123,7 @@ def set_password(self, password=None): def _get_bridge_script(self): """ + iptables has been replaced, use _get_bridge_script_nftables If self.bridge is True, then enable a bridge from eth0 to wlan0 """ if self.bridge: @@ -137,6 +138,35 @@ def _get_bridge_script(self): else: return "" + def _get_bridge_script_nftables(self): + """ + If self.bridge is True, then enable a bridge from eth0 to wlan0 + """ + if self.bridge: + script = [] + script += ["sudo sed -i 's/#net\\.ipv4\\.ip_forward=1/net.ipv4.ip_forward=1/' /etc/sysctl.conf"] + script += ['sudo nft add table ip filter'] + script += ['sudo nft add chain ip filter INPUT "{ type filter hook input priority 0; policy accept; }"'] + script += ['sudo nft add chain ip filter FORWARD "{ type filter hook forward priority 0; policy accept; }"'] + script += ['sudo nft add chain ip filter OUTPUT "{ type filter hook output priority 0; policy accept; }"'] + script += ['sudo nft add rule ip filter FORWARD iifname "eth0" oifname "wlan0" counter accept'] + script += ['sudo nft add rule ip filter FORWARD iifname "wlan0" oifname "eth0" ct state related,established counter accept'] + script += ['sudo nft add table ip nat'] + script += ['sudo nft add chain ip nat PREROUTING "{ type nat hook prerouting priority -100; policy accept; }"'] + script += ['sudo nft add chain ip nat INPUT "{ type nat hook input priority 100; policy accept; }"'] + script += ['sudo nft add chain ip nat OUTPUT "{ type nat hook output priority -100; policy accept; }"'] + script += ['sudo nft add chain ip nat POSTROUTING "{ type nat hook postrouting priority 100; policy accept; }"'] + script += ['sudo nft add rule ip nat POSTROUTING oifname "wlan0" counter masquerade'] + script += [''] + # sudo nft list ruleset + script += ['sudo nft list ruleset | sudo tee -a /etc/nftables.conf'] + script += ['sudo systemctl stop nftables'] + script += ['sudo systemctl enable nftables'] + script += ['sudo systemctl start nftables'] + return '\n'.join(script) + else: + return "" + def _get_password_script(self): # # BUG: hash should be hash_str, you can not use hash and .hash as function @@ -248,9 +278,9 @@ def get(self, verbose=False): cp /etc/ssh/sshd_config /etc/ssh/sshd_config.orig echo 'PasswordAuthentication no' >>/etc/ssh/sshd_config systemctl enable ssh -{self._get_bridge_script()} {self._get_password_script()} {self._get_wifi_config()} +{self._get_bridge_script_nftables()} rm -f /etc/xdg/autostart/piwiz.desktop rm -f /etc/localtime echo "{self.timezone}" >/etc/timezone diff --git a/cloudmesh/burn/sdcard.py b/cloudmesh/burn/sdcard.py index c8e35d19..cfbb7f0b 100644 --- a/cloudmesh/burn/sdcard.py +++ b/cloudmesh/burn/sdcard.py @@ -472,6 +472,7 @@ def mount(self, device=None, card_os="raspberry"): try: Console.ok(f"mounting {device}") os.system('sudo sync') # flush any pending/in-process writes + os.system(f"sudo eject {device}") os.system(f"sudo eject -t {device}") os.system('sudo sync') # flush any pending/in-process writes @@ -479,7 +480,7 @@ def mount(self, device=None, card_os="raspberry"): device_basename = os.path.basename(device) part1 = False part2 = False - for i in range(10): + for i in range(20): result = Shell.run('lsblk') if device_basename in result.split(): for line in result.splitlines(): diff --git a/cloudmesh/burn/ubuntu/configure.py b/cloudmesh/burn/ubuntu/configure.py index b7e2bd99..30ac91a2 100644 --- a/cloudmesh/burn/ubuntu/configure.py +++ b/cloudmesh/burn/ubuntu/configure.py @@ -5,6 +5,8 @@ from cloudmesh.inventory.inventory import Inventory from cloudmesh.common.Shell import Shell from cloudmesh.common.util import path_expand +from cloudmesh.common.util import banner +from cloudmesh.burn.image import Image class Configure: @@ -29,7 +31,7 @@ class Configure: """ KEY_DIR = path_expand('~/.cloudmesh/cmburn') - def __init__(self, inventory=None, cluster=None, debug=False): + def __init__(self, inventory=None, cluster=None, debug=False, download_images=False): self.debug = debug self.manager_public_key = None # Populated by self.generate_ssh_key @@ -45,6 +47,36 @@ def __init__(self, inventory=None, cluster=None, debug=False): self.manager_public_key = None + managers = self.inventory.find(service='manager') + workers = self.inventory.find(service='worker') + configs = managers + workers + # Create dict for them for easy lookup + self.configs = dict((config['host'], config) for config in configs) + if download_images: + self.get_images() + + def get_images(self): + """ + Downloads all tags found in self.configs + """ + tags = set() + for config in self.configs.values(): + try: + tags.add(config['tag']) + except KeyError as e: + Console.warning(f'Could not find tag for {config["host"]}. Skipping') + + banner("Downloading Images", figlet=True) + + image = Image() + + for tag in tags: + Console.info(f'Attempting to download {tag}') + res = image.fetch(tag=[tag]) + if not res: + Console.error('Failed Image Fetch.') + raise Exception('Failed Image Fetch') + def build_user_data(self, name=None, with_defaults=True, country=None, add_manager_key=False, upgrade=False, with_bridge=False): """ @@ -77,7 +109,7 @@ def build_user_data(self, name=None, with_defaults=True, country=None, if with_defaults: user_data.with_locale().with_net_tools().with_packages( - packages='avahi-daemon') + packages=['avahi-daemon','libraspberrypi-bin']) if upgrade: user_data.with_package_update().with_package_upgrade() if hostname: @@ -101,7 +133,10 @@ def build_user_data(self, name=None, with_defaults=True, country=None, .with_runcmd(cmd='sudo rm /boot/firmware/id_rsa.pub')\ .with_runcmd(cmd='sudo rm /boot/firmware/id_rsa') if with_bridge: - user_data.with_access_point_bridge() + user_data.with_access_point_bridge_nftables() + if with_defaults: + # disable cloud-init on subsequent boots + user_data.with_runcmd(cmd='sudo touch /etc/cloud/cloud-init.disabled') if self.debug: Console.info(f'User data for {name}:\n' + str(user_data)) diff --git a/cloudmesh/burn/ubuntu/userdata.py b/cloudmesh/burn/ubuntu/userdata.py index a89a5103..8e2b97d6 100644 --- a/cloudmesh/burn/ubuntu/userdata.py +++ b/cloudmesh/burn/ubuntu/userdata.py @@ -190,6 +190,36 @@ def with_access_point_bridge(self, priv_interface='eth0', ext_interface='wlan0') return self + def with_access_point_bridge_nftables(self, priv_interface='eth0', ext_interface='wlan0'): + """ + Updated version of the previous function that uses nftables instead of iptables. + + Uses iptables to configure an access point bridge where devices on priv_interface + can route internet traffice through the priv_interface of the device towards ext_interface + + Often, this is practical for providing an entire cluster internet through a centralized point (the manager) + """ + self.with_packages(packages=["nftables"]) + self.with_runcmd(cmd="sudo sysctl -w net.ipv4.ip_forward=1")\ + .with_runcmd(cmd="sudo sed -i 's/#net\\.ipv4\\.ip_forward=1/net.ipv4.ip_forward=1/' /etc/sysctl.conf")\ + .with_runcmd(cmd='sudo nft add table ip filter')\ + .with_runcmd(cmd='sudo nft add chain ip filter INPUT "{ type filter hook input priority 0; policy accept; }"')\ + .with_runcmd(cmd='sudo nft add chain ip filter FORWARD "{ type filter hook forward priority 0; policy accept; }"')\ + .with_runcmd(cmd='sudo nft add chain ip filter OUTPUT "{ type filter hook output priority 0; policy accept; }"')\ + .with_runcmd(cmd='sudo nft add rule ip filter FORWARD iifname "eth0" oifname "wlan0" counter accept') \ + .with_runcmd(cmd='sudo nft add rule ip filter FORWARD iifname "wlan0" oifname "eth0" ct state related,established counter accept') \ + .with_runcmd(cmd='sudo nft add table ip nat') \ + .with_runcmd(cmd='sudo nft add chain ip nat PREROUTING "{ type nat hook prerouting priority -100; policy accept; }"') \ + .with_runcmd(cmd='sudo nft add chain ip nat INPUT "{ type nat hook input priority 100; policy accept; }"') \ + .with_runcmd(cmd='sudo nft add chain ip nat OUTPUT "{ type nat hook output priority -100; policy accept; }"') \ + .with_runcmd(cmd='sudo nft add chain ip nat POSTROUTING "{ type nat hook postrouting priority 100; policy accept; }"') \ + .with_runcmd(cmd='sudo nft add rule ip nat POSTROUTING oifname "wlan0" counter masquerade') \ + .with_runcmd(cmd='sudo nft list ruleset | sudo tee -a /etc/nftables.conf') \ + .with_runcmd(cmd='sudo systemctl stop nftables') \ + .with_runcmd(cmd='sudo systemctl enable nftables') \ + .with_runcmd(cmd='sudo systemctl start nftables') + return self + def with_write_files(self, encoding=None, content=None, diff --git a/cloudmesh/burn/usb.py b/cloudmesh/burn/usb.py index efedf9d9..d1f969ce 100644 --- a/cloudmesh/burn/usb.py +++ b/cloudmesh/burn/usb.py @@ -335,9 +335,9 @@ def check_for_readers(): raise ValueError("No card found") elif len(readers) > 1: print() - Console.error("At this time we only support one SDCard " - "reader/writer for MacOS") - raise ValueError("Too many cards found") + # Console.error("At this time we only support one SDCard " + # "reader/writer for MacOS") + # raise ValueError("Too many cards found") @staticmethod def get_dev_from_diskutil(): diff --git a/docs/tutorial_scrub.md b/docs/tutorial_scrub.md new file mode 100644 index 00000000..b57dc93f --- /dev/null +++ b/docs/tutorial_scrub.md @@ -0,0 +1,84 @@ + +## Working Pi Burning Tutorials: + +Burning a set of pre-configured Raspberry OS cards for Raspberry Pis with Wifi Access + +- https://cloudmesh.github.io/pi/tutorial/raspberry-burn/ + +Burning a set of Ubuntu Server Cards for Raspberry Pis with Internet Access + +- https://cloudmesh.github.io/pi/tutorial/ubuntu-burn/ + +DRAFT: Burning a pre-configured RaspberryOS Cluster on Windows 10 + +- https://cloudmesh.github.io/pi/tutorial/raspberry-burn-windows/ +- NOTE: Works for raspi OS only + +Burning a set of pre-configured SD cards with a GUI for Raspberry Pis with Wifi Access + +- https://cloudmesh.github.io/pi/tutorial/gui-burn/ +- NOTE: Based only on code inspection I expect this to still work, or be easiyly fixed. + + +### Hackaday.io + +Easy Raspberry PI Cluster Setup with Cloudmesh from MacOS +- https://hackaday.io/project/177904-headless-rasbery-pi-cluster-from-macs/details + +### Github.com + +README.md +- https://github.com/cloudmesh/cloudmesh-pi-burn + + +## FIX Prepared Burning Tutorials: + +### Opensource.com + +Rapidly configure SD cards for your Raspberry Pi cluster +- https://opensource.com/article/21/3/raspberry-pi-cluster +- STATUS: DEFUNCT +- Expected BURNER OS: Linux and Mac +- REASON: uses `cms burn cluster...` command which is out of date + +### Medium.com + +Easy Raspberry PI Cluster Setup with Cloudmesh from MacOS +- https://laszewski.medium.com/easy-raspberry-pi-cluster-setup-with-cloudmesh-from-macos-e160ac848bf +- STATUS: DEFUNCT +- Expected BURNER OS: Mac +- REASON: uses `cms burn cluster...` command which is out of date + +## DEFUNCT tutorials. Need to point to new tutorials: + +### Hackaday.io + +Preconfigured SDCards for Raspberry Pi Clusters +- https://hackaday.io/project/177874-preconfigured-sdcards-for-raspberry-pi-clusters +- STATUS: DEFUNCT +- Expected BURNER OS: Linux and Mac, also provides manager pi imaging instructions to burn from Pi +- REASON: uses `cms burn create..` which is out of date + +### Medium.com + +Easy Raspberry PI Cluster Setup with Cloudmesh SDCard Burner +- https://laszewski.medium.com/easy-raspberry-pi-cluster-setup-with-cloudmesh-sdcard-burner-a2035dfea22 +- STATUS: DEFUNCT +- Expected BURNER OS: Linux and Mac, also provides manager pi imaging instructions to burn from Pi +- REASON: uses `cms burn create..` which is out of date + +## Removed Pi Burning Tutorials: + +### Pi-planet.org + +Easy Raspberry PI Cluster Setup with Cloudmesh from MacOS +- https://cloudmesh.github.io/pi/tutorial/sdcard-burn-pi-headless/ +- STATUS: DEFUNCT +- Expected BURNER OS: Mac +- REASON: uses `cms burn cluster...` command which is out of date + +Easy Raspberry PI Cluster Setup with Cloudmesh SDCard Burner +- https://cloudmesh.github.io/pi/tutorial/sdcard-burn-pi-as-burner/ +- STATUS: DEFUNCT +- Expected BURNER OS: Linux and Mac, also provides manager pi imaging instructions to burn from Pi +- REASON: uses `cms burn create..` which is out of date diff --git a/tests/test_06_configure.py b/tests/test_06_configure.py index 89e5c323..fe33187c 100644 --- a/tests/test_06_configure.py +++ b/tests/test_06_configure.py @@ -17,8 +17,8 @@ def test_build_user_data(self): HEADING() inv_file = '~/.cloudmesh/config_test.yaml' inv = Inventory(inv_file) - inv.add(host='test_host1', keyfile='~/.ssh/id_rsa.pub', service='manager') - inv.add(host='test_host2', service='worker') + inv.add_directory(host='test_host1', keyfile='~/.ssh/id_rsa.pub', service='manager') + inv.add_directory(host='test_host2', service='worker') inv.save() c = Configure(inventory=inv_file) @@ -52,8 +52,8 @@ def test_build_networkdata(self): HEADING() inv_file = '~/.cloudmesh/config_test.yaml' inv = Inventory(inv_file) - inv.add(host='test_host1', keyfile='~/.ssh/id_rsa.pub', service='manager', ip='10.1.1.11', router='10.1.1.1', dns=['8.8.8.8', '8.8.4.4']) - inv.add(host='test_host2', service='worker', ip='10.1.1.10', router='10.1.1.1', dns=['8.8.8.8', '8.8.4.4']) + inv.add_directory(host='test_host1', keyfile='~/.ssh/id_rsa.pub', service='manager', ip='10.1.1.11', router='10.1.1.1', dns=['8.8.8.8', '8.8.4.4']) + inv.add_directory(host='test_host2', service='worker', ip='10.1.1.10', router='10.1.1.1', dns=['8.8.8.8', '8.8.4.4']) inv.save() c = Configure(inventory=inv_file) @@ -81,7 +81,7 @@ def test_key_gen(self): HEADING() inv_file = '~/.cloudmesh/config_test.yaml' inv = Inventory(inv_file) - inv.add(host='test_host1', keyfile='~/.ssh/id_rsa.pub', service='manager') + inv.add_directory(host='test_host1', keyfile='~/.ssh/id_rsa.pub', service='manager') inv.save() c = Configure(inventory=inv_file)