Skip to content

config

DockerConfig

Bases: BaseModel

Contains information about the docker images to build.

Attributes:

Name Type Description
global_env dict[str, str] | None

...
default_runner str

...
architectures dict[str, str | list[str]] | None

...
platforms dict[str, str] | None

...
container dict[str, Any] | None

...
base_path str

...
docker_registry str

...
extra_build_steps list[dict[str, Any]] | None

...
workflow_inputs dict[str, GithubWorkflowInput] | None

...
max_parallel_manifests int | None

...
images list[DockerImage]

...
use_buildx bool

...
Source code in m/ci/docker/config.py 
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
class DockerConfig(BaseModel):
    """Contains information about the docker images to build."""

    # additional environment variables to inject globally.
    global_env: dict[str, str] | None = None

    # default runner to use when creating blueprints and manifests
    default_runner: str = 'ubuntu-22.04'

    # A map of the architectures to build. It maps say `amd64` to a Github
    # runner that will build the image for that architecture.
    #    amd64: Ubuntu 20.04
    architectures: dict[str, str | list[str]] | None

    # A map of the platforms to build. It maps say `amd64` to a valid buildx
    # supported platform. Using this allows us to build multi-arch images using
    # buildx in an environment that may not have the necessary architecture.
    # For instance: 'amd64: linux/amd64'
    platforms: dict[str, str] | None = None

    # Freeform object to allow us to specify a container in which to run
    # the docker commands.
    # https://docs.github.com/en/actions/using-jobs/running-jobs-in-a-container
    # Note that this requires the mounting the following volume
    # - /var/run/docker.sock:/var/run/docker.sock
    # This may be needed for self-hosted runners that may not have python or pip.
    container: dict[str, Any] | None = None

    # Base path used to locate docker files. Defaults to `.` (root of project)
    # but may be changed a specific directory.
    base_path: str = '.'

    # Name of the docker registry to push the images to. For Github container
    # registry make sure to also include the github owner. For instance:
    # ghcr.io/owner
    docker_registry: str

    # When executing docker build commands we may need to obtain external tokens
    # via other github actions. These can be injected here. We can see those
    # steps in the github workflow file before the actual docker shell scripts
    # are run.
    extra_build_steps: list[dict[str, Any]] | None = None

    # Additional inputs to the github workflow file.
    workflow_inputs: dict[str, GithubWorkflowInput] | None = None

    # Maximum number of parallel manifests to build.
    # https://docs.github.com/en/actions/using-jobs/using-a-matrix-for-your-jobs#defining-the-maximum-number-of-concurrent-jobs
    max_parallel_manifests: int | None = None

    # list of images to build
    images: list[DockerImage]

    # https://docs.docker.com/reference/cli/docker/buildx/imagetools/create/
    use_buildx: bool = True

    def makefile_targets(self: 'DockerConfig', files: FileNames) -> str:
        """Create the Makefile targets to trigger the local builds.

        Args:
            files: Instance of FileNames to obtain the names of the scripts.

        Returns:
            A string with the Makefile targets.
        """
        m_dir = files.m_dir
        lines: list[str] = [
            'define m_env',
            f'\t$(eval include {m_dir}/.m/m_env.sh)',
            f'\t$(eval $(cut -d= -f1 {m_dir}/.m/m_env.sh))',
            'endef',
            '',
            'm-env:',
            f'\tmkdir -p {m_dir}/.m && m ci env --bashrc > {m_dir}/.m/m_env.sh',
            '',
            'm-blueprints: m-env',
            '\t$(call m_env)',
            f'\tm blueprints --skip-makefile --skip-workflow {m_dir}',
            f'\tchmod +x {m_dir}/.m/blueprints/local/*.sh\n',
        ]
        for index, img in enumerate(self.images):
            name = img.image_name
            img_file = files.local_file(f'{name}.build.sh')
            previous_img = (
                self.images[index - 1].image_name
                if index > 0
                else None
            )
            dep = f' dev-{previous_img}' if previous_img else ' m-blueprints'
            lines.append(f'dev-{name}:{dep}')
            lines.append('\t$(call m_env)')
            lines.append(f'\t{img_file}\n')
        return '\n'.join(lines)

    def update_makefile(self: 'DockerConfig', files: FileNames) -> Res[int]:
        """Update the Makefile with the docker images targets.

        Args:
            files: Instance of FileNames to obtain the names of the scripts.

        Returns:
            None if successful, else an issue.
        """
        return rw.insert_to_file(
            files.makefile,
            '\n# START: M-DOCKER-IMAGES\n',
            self.makefile_targets(files),
            '\n# END: M-DOCKER-IMAGES\n',
        )

    def update_github_workflow(
        self: 'DockerConfig',
        files: FileNames,
    ) -> Res[int]:
        """Update the github workflow with the docker images targets.

        Args:
            files: Instance of FileNames to obtain the names of the scripts.

        Returns:
            None if successful, else an issue.
        """
        global_env: dict[str, str] = self.global_env or {}
        multi_workflow = MultiWorkflow(
            m_dir=files.m_dir,
            ci_dir=files.ci_dir,
            global_env=global_env,
            default_runner=self.default_runner,
            architectures=self.architectures or {},
            platforms=self.platforms,
            images=self.images,
            extra_build_steps=self.extra_build_steps,
            docker_registry=self.docker_registry,
            extra_inputs=self.workflow_inputs,
            max_parallel_manifests=self.max_parallel_manifests,
            container=self.container,
            use_buildx=self.use_buildx,
        )
        single_workflow = SingleWorkflow(
            m_dir=files.m_dir,
            ci_dir=files.ci_dir,
            global_env=global_env,
            default_runner=self.default_runner,
            images=self.images,
            extra_build_steps=self.extra_build_steps,
            extra_inputs=self.workflow_inputs,
            docker_registry=self.docker_registry,
            container=self.container,
        )
        workflow = multi_workflow if self.architectures else single_workflow
        return rw.write_file(files.gh_workflow, str(workflow))

    def write_local_steps(
        self: 'DockerConfig',
        files: FileNames,
        m_env: MEnvDocker,
    ) -> Res[None]:
        """Write local entry point files.

        Args:
            files: The FileNames instance with the file names.
            m_env: The MEnvDocker instance with the environment variables.

        Returns:
            None if successful, else an issue.
        """
        issues: list[dict] = []
        for img in self.images:
            write_res = _write_local_step(files, img, m_env)
            _append_issue(write_res, issues)
        if issues:
            return issue(
                'write_local_steps_failure',
                context={'issues': issues},
            )
        return Good(None)

    def write_ci_steps(
        self: 'DockerConfig',
        files: FileNames,
        m_env: MEnvDocker,
    ) -> Res[None]:
        """Write ci entry point files.

        Args:
            files: The FileNames instance with the file names.
            m_env: The MEnvDocker instance with the environment variables.

        Returns:
            None if successful, else an issue.
        """
        issues: list[dict] = []
        registry = self.docker_registry
        cache_script = create_cache_script(m_env.cache_from_pr, registry)
        push_script = (
            create_push_script(registry)
            if self.architectures
            else create_push_script_tags(registry, m_env.m_tag)
        )
        script_results = [
            rw.write_file(f'{files.ci_dir}/_find-cache.sh', cache_script),
            rw.write_file(f'{files.ci_dir}/_push-image.sh', push_script),
        ]
        for script_res in script_results:
            _append_issue(script_res, issues)
        for img in self.images:
            file_name = f'{files.ci_dir}/{img.image_name}.build.sh'
            write_res = _write_build_script(file_name, img, m_env)
            _append_issue(write_res, issues)
            if self.use_buildx:
                file_name = f'{files.ci_dir}/{img.image_name}.manifest.sh'
                write_res = _write_manifest_script(file_name, img, m_env)
                _append_issue(write_res, issues)
        if issues:
            return issue(
                'write_ci_steps_failure',
                context={'issues': issues},
            )
        return Good(None)

    def write_ci_manifest_info(
        self: 'DockerConfig',
        files: FileNames,
        m_env: MEnvDocker,
    ) -> Res[None]:
        """Write ci entry point files.

        Args:
            files: The FileNames instance with the file names.
            m_env: The MEnvDocker instance with the environment variables.

        Returns:
            None if successful, else an issue.
        """
        if not self.architectures:
            return Good(None)
        m_tag = m_env.m_tag
        if not m_tag and os.environ.get('CI') != 'true':
            logger.warning('M_TAG not found in non-CI environment. Using 1.1.1')
            m_tag = '1.1.1'
        names = [img.image_name for img in self.images]
        tags = [m_tag, *docker_tags(m_tag)]
        names_json = json.dumps(names, separators=(',', ':'))
        tags_json = json.dumps(tags, separators=(',', ':'))
        files_res = [
            rw.write_file(f'{files.ci_dir}/_image-names.json', names_json),
            rw.write_file(f'{files.ci_dir}/_image-tags.json', tags_json),
        ]
        issues: list[dict] = []
        for file_res in files_res:
            _append_issue(file_res, issues)
        if issues:
            return issue(
                'write_ci_manifest_info_failure',
                context={'issues': issues},
            )
        return Good(None)

    def write_blueprints(
        self: 'DockerConfig',
        m_dir: str,
        m_env: MEnvDocker,
    ) -> Res[None]:
        """Write entry point files.

        Writes shell files for both local and ci. Updates the Makefile and
        github workflow file.

        Args:
            m_dir: The m directory.
            m_env: The MEnvDocker instance with the environment variables.

        Returns:
            None if successful, else an issue.
        """
        files = FileNames.create_instance(m_dir)
        return one_of(lambda: [
            None
            for _ in self.write_local_steps(files, m_env)
            for _ in self.write_ci_steps(files, m_env)
            for _ in self.write_ci_manifest_info(files, m_env)
        ])

makefile_targets(files)

Create the Makefile targets to trigger the local builds.

Parameters:

Name Type Description Default
files FileNames

Instance of FileNames to obtain the names of the scripts.

 required

Returns:

Type Description
str

A string with the Makefile targets.
Source code in m/ci/docker/config.py 
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
def makefile_targets(self: 'DockerConfig', files: FileNames) -> str:
    """Create the Makefile targets to trigger the local builds.

    Args:
        files: Instance of FileNames to obtain the names of the scripts.

    Returns:
        A string with the Makefile targets.
    """
    m_dir = files.m_dir
    lines: list[str] = [
        'define m_env',
        f'\t$(eval include {m_dir}/.m/m_env.sh)',
        f'\t$(eval $(cut -d= -f1 {m_dir}/.m/m_env.sh))',
        'endef',
        '',
        'm-env:',
        f'\tmkdir -p {m_dir}/.m && m ci env --bashrc > {m_dir}/.m/m_env.sh',
        '',
        'm-blueprints: m-env',
        '\t$(call m_env)',
        f'\tm blueprints --skip-makefile --skip-workflow {m_dir}',
        f'\tchmod +x {m_dir}/.m/blueprints/local/*.sh\n',
    ]
    for index, img in enumerate(self.images):
        name = img.image_name
        img_file = files.local_file(f'{name}.build.sh')
        previous_img = (
            self.images[index - 1].image_name
            if index > 0
            else None
        )
        dep = f' dev-{previous_img}' if previous_img else ' m-blueprints'
        lines.append(f'dev-{name}:{dep}')
        lines.append('\t$(call m_env)')
        lines.append(f'\t{img_file}\n')
    return '\n'.join(lines)

update_github_workflow(files)

Update the github workflow with the docker images targets.

Parameters:

Name Type Description Default
files FileNames

Instance of FileNames to obtain the names of the scripts.

 required

Returns:

Type Description
Res[int]

None if successful, else an issue.
Source code in m/ci/docker/config.py 
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
def update_github_workflow(
    self: 'DockerConfig',
    files: FileNames,
) -> Res[int]:
    """Update the github workflow with the docker images targets.

    Args:
        files: Instance of FileNames to obtain the names of the scripts.

    Returns:
        None if successful, else an issue.
    """
    global_env: dict[str, str] = self.global_env or {}
    multi_workflow = MultiWorkflow(
        m_dir=files.m_dir,
        ci_dir=files.ci_dir,
        global_env=global_env,
        default_runner=self.default_runner,
        architectures=self.architectures or {},
        platforms=self.platforms,
        images=self.images,
        extra_build_steps=self.extra_build_steps,
        docker_registry=self.docker_registry,
        extra_inputs=self.workflow_inputs,
        max_parallel_manifests=self.max_parallel_manifests,
        container=self.container,
        use_buildx=self.use_buildx,
    )
    single_workflow = SingleWorkflow(
        m_dir=files.m_dir,
        ci_dir=files.ci_dir,
        global_env=global_env,
        default_runner=self.default_runner,
        images=self.images,
        extra_build_steps=self.extra_build_steps,
        extra_inputs=self.workflow_inputs,
        docker_registry=self.docker_registry,
        container=self.container,
    )
    workflow = multi_workflow if self.architectures else single_workflow
    return rw.write_file(files.gh_workflow, str(workflow))

update_makefile(files)

Update the Makefile with the docker images targets.

Parameters:

Name Type Description Default
files FileNames

Instance of FileNames to obtain the names of the scripts.

 required

Returns:

Type Description
Res[int]

None if successful, else an issue.
Source code in m/ci/docker/config.py 
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
def update_makefile(self: 'DockerConfig', files: FileNames) -> Res[int]:
    """Update the Makefile with the docker images targets.

    Args:
        files: Instance of FileNames to obtain the names of the scripts.

    Returns:
        None if successful, else an issue.
    """
    return rw.insert_to_file(
        files.makefile,
        '\n# START: M-DOCKER-IMAGES\n',
        self.makefile_targets(files),
        '\n# END: M-DOCKER-IMAGES\n',
    )

write_blueprints(m_dir, m_env)

Write entry point files.

Writes shell files for both local and ci. Updates the Makefile and github workflow file.

Parameters:

Name Type Description Default
m_dir str

The m directory.

 required
m_env MEnvDocker

The MEnvDocker instance with the environment variables.

 required

Returns:

Type Description
Res[None]

None if successful, else an issue.
Source code in m/ci/docker/config.py 
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
def write_blueprints(
    self: 'DockerConfig',
    m_dir: str,
    m_env: MEnvDocker,
) -> Res[None]:
    """Write entry point files.

    Writes shell files for both local and ci. Updates the Makefile and
    github workflow file.

    Args:
        m_dir: The m directory.
        m_env: The MEnvDocker instance with the environment variables.

    Returns:
        None if successful, else an issue.
    """
    files = FileNames.create_instance(m_dir)
    return one_of(lambda: [
        None
        for _ in self.write_local_steps(files, m_env)
        for _ in self.write_ci_steps(files, m_env)
        for _ in self.write_ci_manifest_info(files, m_env)
    ])

write_ci_manifest_info(files, m_env)

Write ci entry point files.

Parameters:

Name Type Description Default
files FileNames

The FileNames instance with the file names.

 required
m_env MEnvDocker

The MEnvDocker instance with the environment variables.

 required

Returns:

Type Description
Res[None]

None if successful, else an issue.
Source code in m/ci/docker/config.py 
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
def write_ci_manifest_info(
    self: 'DockerConfig',
    files: FileNames,
    m_env: MEnvDocker,
) -> Res[None]:
    """Write ci entry point files.

    Args:
        files: The FileNames instance with the file names.
        m_env: The MEnvDocker instance with the environment variables.

    Returns:
        None if successful, else an issue.
    """
    if not self.architectures:
        return Good(None)
    m_tag = m_env.m_tag
    if not m_tag and os.environ.get('CI') != 'true':
        logger.warning('M_TAG not found in non-CI environment. Using 1.1.1')
        m_tag = '1.1.1'
    names = [img.image_name for img in self.images]
    tags = [m_tag, *docker_tags(m_tag)]
    names_json = json.dumps(names, separators=(',', ':'))
    tags_json = json.dumps(tags, separators=(',', ':'))
    files_res = [
        rw.write_file(f'{files.ci_dir}/_image-names.json', names_json),
        rw.write_file(f'{files.ci_dir}/_image-tags.json', tags_json),
    ]
    issues: list[dict] = []
    for file_res in files_res:
        _append_issue(file_res, issues)
    if issues:
        return issue(
            'write_ci_manifest_info_failure',
            context={'issues': issues},
        )
    return Good(None)

write_ci_steps(files, m_env)

Write ci entry point files.

Parameters:

Name Type Description Default
files FileNames

The FileNames instance with the file names.

 required
m_env MEnvDocker

The MEnvDocker instance with the environment variables.

 required

Returns:

Type Description
Res[None]

None if successful, else an issue.
Source code in m/ci/docker/config.py 
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
def write_ci_steps(
    self: 'DockerConfig',
    files: FileNames,
    m_env: MEnvDocker,
) -> Res[None]:
    """Write ci entry point files.

    Args:
        files: The FileNames instance with the file names.
        m_env: The MEnvDocker instance with the environment variables.

    Returns:
        None if successful, else an issue.
    """
    issues: list[dict] = []
    registry = self.docker_registry
    cache_script = create_cache_script(m_env.cache_from_pr, registry)
    push_script = (
        create_push_script(registry)
        if self.architectures
        else create_push_script_tags(registry, m_env.m_tag)
    )
    script_results = [
        rw.write_file(f'{files.ci_dir}/_find-cache.sh', cache_script),
        rw.write_file(f'{files.ci_dir}/_push-image.sh', push_script),
    ]
    for script_res in script_results:
        _append_issue(script_res, issues)
    for img in self.images:
        file_name = f'{files.ci_dir}/{img.image_name}.build.sh'
        write_res = _write_build_script(file_name, img, m_env)
        _append_issue(write_res, issues)
        if self.use_buildx:
            file_name = f'{files.ci_dir}/{img.image_name}.manifest.sh'
            write_res = _write_manifest_script(file_name, img, m_env)
            _append_issue(write_res, issues)
    if issues:
        return issue(
            'write_ci_steps_failure',
            context={'issues': issues},
        )
    return Good(None)

write_local_steps(files, m_env)

Write local entry point files.

Parameters:

Name Type Description Default
files FileNames

The FileNames instance with the file names.

 required
m_env MEnvDocker

The MEnvDocker instance with the environment variables.

 required

Returns:

Type Description
Res[None]

None if successful, else an issue.
Source code in m/ci/docker/config.py 
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
def write_local_steps(
    self: 'DockerConfig',
    files: FileNames,
    m_env: MEnvDocker,
) -> Res[None]:
    """Write local entry point files.

    Args:
        files: The FileNames instance with the file names.
        m_env: The MEnvDocker instance with the environment variables.

    Returns:
        None if successful, else an issue.
    """
    issues: list[dict] = []
    for img in self.images:
        write_res = _write_local_step(files, img, m_env)
        _append_issue(write_res, issues)
    if issues:
        return issue(
            'write_local_steps_failure',
            context={'issues': issues},
        )
    return Good(None)