From 299699ffcbecf157c4869d1f5b70e0555d6a6911 Mon Sep 17 00:00:00 2001 From: sid palas Date: Mon, 13 Feb 2023 09:55:19 -0500 Subject: [PATCH] Add dev workflow README, improve other readmes --- 05-example-web-application/README.md | 6 +- 10-interacting-with-docker-objects/README.md | 114 +++++++++++++----- 11-development-workflow/Makefile | 12 +- 11-development-workflow/README.md | 47 ++++++++ .../api-golang/Dockerfile.dev | 6 +- .../api-node/Dockerfile.dev | 10 -- .../docker-compose-dev.yml | 4 +- .../docker-compose-test.yml | 13 ++ 8 files changed, 151 insertions(+), 61 deletions(-) create mode 100644 11-development-workflow/README.md create mode 100644 11-development-workflow/docker-compose-test.yml diff --git a/05-example-web-application/README.md b/05-example-web-application/README.md index 9dc5fff..6668ac0 100644 --- a/05-example-web-application/README.md +++ b/05-example-web-application/README.md @@ -3,9 +3,9 @@ ![](./readme-assets/app-screenshot.png) ## Minimal 3 tier web application -- React frontend -- Node JS and Golang APIs -- Postgres Database +- **React frontend:** Uses react query to load data from the two apis and display the result +- **Node JS and Golang APIs:** Both have `/` and `/ping` endpoints. `/` queries the Database for the current time, and `/ping` returns `pong` +- **Postgres Database:** An empty PostgreSQL database with no tables or data. Used to show how to set up connectivity. The API applications execute `SELECT NOW() as now;` to determine the current time to return. ![](./readme-assets/tech-stack.png) diff --git a/10-interacting-with-docker-objects/README.md b/10-interacting-with-docker-objects/README.md index 8045c90..153d572 100644 --- a/10-interacting-with-docker-objects/README.md +++ b/10-interacting-with-docker-objects/README.md @@ -8,45 +8,93 @@ You should: ## Images -1) ls -2) build (https://docs.docker.com/engine/reference/commandline/build/) -3) tag -4) pull -5) push -6) rm -7) prune -8) save -9) docker scan (snyk security scan, also show trivy) +`docker image COMMAND`: +``` + build Build an image from a Dockerfile (`docker build` is the same as `docker image build`) + history Show the history of an image + import Import the contents from a tarball to create a filesystem image + inspect Display detailed information on one or more images + load Load an image from a tar archive or STDIN + ls List images + prune Remove unused images + pull Pull an image or a repository from a registry + push Push an image or a repository to a registry + rm Remove one or more images + save Save one or more images to a tar archive (streamed to STDOUT by default) + tag Create a tag TARGET_IMAGE that refers to SOURCE_IMAGE +``` + +### Scanning Images + +Not a `docker image` subcommand, but still something you do with images: + +``` +docker scan IMAGE +``` + +***Note:*** You can also use a 3rd party scanner such as Trivy (https://github.com/aquasecurity/trivy) + +### Signing Images + +Another protection against software supply chain attacks is the ability to uniquely sign specific image tags to ensure an image was created by the entity who signed it. + +``` +docker trust sign IMAGE:TAG +docker trust inspect --pretty IMAGE:TAG +``` ## Containers -1) ls -2) run -3) start -4) attach -5) exec -6) logs -7) top -8) cp -9) stop -10) kill -11) prune -12) export +`docker container COMMAND`: + +``` + attach Attach local standard input, output, and error streams to a running container + commit Create a new image from a container's changes + cp Copy files/folders between a container and the local filesystem + create Create a new container + diff Inspect changes to files or directories on a container's filesystem + exec Run a command in a running container + export Export a container's filesystem as a tar archive + inspect Display detailed information on one or more containers + kill Kill one or more running containers + logs Fetch the logs of a container + ls List containers + pause Pause all processes within one or more containers + port List port mappings or a specific mapping for the container + prune Remove all stopped containers + rename Rename a container + restart Restart one or more containers + rm Remove one or more containers + run Run a command in a new container + start Start one or more stopped containers + stats Display a live stream of container(s) resource usage statistics + stop Stop one or more running containers + top Display the running processes of a container + unpause Unpause all processes within one or more containers + update Update configuration of one or more containers + wait Block until one or more containers stop, then print their exit codes +``` ## Volumes -1) ls -2) create -3) inspect -4) rm -5) prune +`docker volume COMMAND`: +``` + create Create a volume + inspect Display detailed information on one or more volumes + ls List volumes + prune Remove all unused local volumes + rm Remove one or more volumes +``` ## Networks -1) ls -2) create -3) inspect -4) connect -5) disconnect -6) rm -7) prune +`docker network COMMAND`: +``` + connect Connect a container to a network + create Create a network + disconnect Disconnect a container from a network + inspect Display detailed information on one or more networks + ls List networks + prune Remove all unused networks + rm Remove one or more networks +``` diff --git a/11-development-workflow/Makefile b/11-development-workflow/Makefile index b30df71..326f4bc 100644 --- a/11-development-workflow/Makefile +++ b/11-development-workflow/Makefile @@ -1,5 +1,6 @@ DEV_COMPOSE_FILE=docker-compose-dev.yml DEBUG_COMPOSE_FILE=docker-compose-debug.yml +TEST_COMPOSE_FILE=docker-compose-test.yml ### DOCKER COMPOSE COMMANDS @@ -26,14 +27,9 @@ compose-down: ### DOCKERCONTEXT_DIR:=../05-example-web-application/ -DOCKERFILE_DIR:=../10-development-workflow/ - -.PHONY: docker-build-all -docker-build-all: - docker build -t api-node -f ${DOCKERFILE_DIR}/api-node/Dockerfile.dev ${DOCKERCONTEXT_DIR}/api-node/ - docker build -t api-golang -f ${DOCKERFILE_DIR}/api-golang/Dockerfile.dev ${DOCKERCONTEXT_DIR}/api-golang/ +DOCKERFILE_DIR:=../11-development-workflow/ .PHONY: run-tests run-tests: - docker run -t api-golang go test -v ./... - docker run -it api-node npm run test \ No newline at end of file + docker compose -f $(DEV_COMPOSE_FILE) -f $(TEST_COMPOSE_FILE) run --build api-golang + docker compose -f $(DEV_COMPOSE_FILE) -f $(TEST_COMPOSE_FILE) run --build api-node diff --git a/11-development-workflow/README.md b/11-development-workflow/README.md new file mode 100644 index 0000000..94eeadd --- /dev/null +++ b/11-development-workflow/README.md @@ -0,0 +1,47 @@ +# Development Workflow + +## Development Environment + +Because we are running our application within containers, we need a way to quickly iterate and make changes to them. Some of our tactics in `06-building-container-images` help here (e.g. protecting the layer cache) so that images build quickly, but we can do better. + +We want our development environment to have the following attributes: + +1) **Easy/simple to set up:** Using docker compose, we can define the entire environment with a single yaml file. To get started, team members can issue a single command `make compose-up-build` or `make compose-up-build-debug` depending if they want to run the debugger or not. + +2) **Ability to iterate without rebuilding the container image:** In order to avoid having to rebuild the container image with every single change, we can use a bind mount to mount the code from our host into the container filesystem. For example: + +```yml + - type: bind + source: ../05-example-web-application/api-node/ + target: /usr/src/app/ +``` + +3) **Automatic reloading of the application:** + - *React Client:* We are using Vite for the react client which handles this handles this automatically + - *Node API:* We added nodemon as a development dependency and specify the Docker CMD to use it + - *Golang API:* We added a utility called `air` (https://github.com/cosmtrek/air) within `Dockerfile.dev` which watches for changes and rebuild the app automatically. + +4) **Use a debugger:** + - *React Client:* For a react app, you can use the browser developer tools + extensions to debug. I did include `react-query-devtools` to help debug react query specific things. It is also viewed from within the browser. + - *Node API:* To enable debugging for a NodeJS application we can run the app with the `--inspect` flag. The debug session can then be accessed via a websocket on port `9229`. The additional considerations in this case are to specify that the debugger listen for requests from 0.0.0.0 (any) and to publish port `9229` from the container to localhost. + - *Golang API:* To enable remote debugging for a golang application I installed a tool called delve (https://github.com/go-delve/delve) within `./api-golang/Dockerfile.dev`. We then override the command used to run the container to use this tool (see: `docker-compose-debug.yml`) + + --- + + These modifications to the configuration (overridden commands + port publishing) are specified in `docker-compose-debug.yml`. By passing both `docker-compose-dev.yml` AND `docker-compose-debug.yml` to the `docker compose up` command (See: `make compose-up-debug-build`) Docker combines the two files, taking the config from the latter and overlaying it onto the former. + + Both `./api-golang/README.md` and `./api-node/README.md` show a launch.json configuration you can use to connnect to these remote debuggers using VSCode. The key setting is `substitutePath` such that you can set breakpoints on your local system that get recognized within the container. + +5) **Executing tests:** We also need the ability to execute our test suites within containers. Again, we can create a custom `docker-compose-test.yml` overlay which modifies the container commands to execute our tests. To build the api images and execute their tests, you can execute `make run-tests` which will use the `test` compose file along with the `dev` compose file to do so. + +## Continuous Integration + +See `.github/workflows/image-ci.yml` for a basic GitHub Action workflow that builds, scans, tags, and pushes a container image. + +It leverages a few publicly available actions from the marketplace: +1) https://github.com/marketplace/actions/docker-metadata-action (generates tags for the container images) +2) https://github.com/marketplace/actions/docker-login (logs into DockerHub) +3) https://github.com/marketplace/actions/build-and-push-docker-images (builds and pushes the images) +4) https://github.com/marketplace/actions/aqua-security-trivy (scans the images for vulnerabilities) + +If you want to build out more advanced CI workflows I recommend looking at Bret Fisher's `Automation with Docker for CI/CD Workflows` repo (https://github.com/BretFisher/docker-cicd-automation). It has many great examples of the types of things you might want to do with Docker in a CI/CD pipeline! \ No newline at end of file diff --git a/11-development-workflow/api-golang/Dockerfile.dev b/11-development-workflow/api-golang/Dockerfile.dev index dc94ede..28b6155 100644 --- a/11-development-workflow/api-golang/Dockerfile.dev +++ b/11-development-workflow/api-golang/Dockerfile.dev @@ -1,7 +1,4 @@ -# Pin specific version for stability -# using bullseye instead of alpine because of: -## runtime/cgo -## cgo: C compiler "gcc" not found: exec: "gcc": executable file not found in $PATH +# Using bullseye instead of alpine because debugger didnt work in alpine FROM golang:1.19-bullseye WORKDIR /app @@ -12,7 +9,6 @@ RUN go install github.com/cosmtrek/air@latest # Install delve for debugging RUN go install github.com/go-delve/delve/cmd/dlv@latest -# Copy only files required to install dependencies (better layer caching) COPY go.mod go.sum ./ RUN go mod download diff --git a/11-development-workflow/api-node/Dockerfile.dev b/11-development-workflow/api-node/Dockerfile.dev index 69b3a25..b0577b2 100644 --- a/11-development-workflow/api-node/Dockerfile.dev +++ b/11-development-workflow/api-node/Dockerfile.dev @@ -1,25 +1,15 @@ -# Pin specific version for stability -# Use alpine for reduced image size FROM node:19.4-alpine as dev -# Specify working directory other than / WORKDIR /usr/src/app -# Copy only files required to install -# dependencies (better layer caching) COPY package*.json ./ -# Install only production dependencies -# Use cache mount to speed up install of existing dependencies RUN --mount=type=cache,target=/usr/src/app/.npm \ npm set cache /usr/src/app/.npm && \ npm install -# Copy remaining source code AFTER installing dependencies. -# Again, copy only the necessary files COPY . . -# Indicate expected port EXPOSE 3000 CMD [ "npm", "run", "dev" ] diff --git a/11-development-workflow/docker-compose-dev.yml b/11-development-workflow/docker-compose-dev.yml index 6fc6112..cb6e938 100644 --- a/11-development-workflow/docker-compose-dev.yml +++ b/11-development-workflow/docker-compose-dev.yml @@ -18,7 +18,7 @@ services: api-node: build: context: ../05-example-web-application/api-node/ - dockerfile: ../../10-development-workflow/api-node/Dockerfile.dev + dockerfile: ../../11-development-workflow/api-node/Dockerfile.dev target: dev volumes: - type: bind @@ -37,7 +37,7 @@ services: api-golang: build: context: ../05-example-web-application/api-golang/ - dockerfile: ../../10-development-workflow/api-golang/Dockerfile.dev + dockerfile: ../../11-development-workflow/api-golang/Dockerfile.dev volumes: - type: bind source: ../05-example-web-application/api-golang/ diff --git a/11-development-workflow/docker-compose-test.yml b/11-development-workflow/docker-compose-test.yml new file mode 100644 index 0000000..64baf27 --- /dev/null +++ b/11-development-workflow/docker-compose-test.yml @@ -0,0 +1,13 @@ +# Overlay configuration to enable debuggers +services: + api-node: + command: + - "npm" + - "run" + - "test" + api-golang: + command: + - "go" + - "test" + - "-v" + - "./..." \ No newline at end of file