-
Notifications
You must be signed in to change notification settings - Fork 8
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge branch 'dev' into additional-partitions
- Loading branch information
Showing
15 changed files
with
403 additions
and
36 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,60 @@ | ||
| """ | ||
| Handles the schema validation for BiBiGrid's configuration yml. | ||
| """ | ||
|
|
||
| from schema import Schema, Optional, Or, SchemaError | ||
|
|
||
| # Define the schema for the configuration file | ||
| master_schema = Schema( | ||
| {'infrastructure': str, 'cloud': str, 'sshUser': str, Or('subnet', 'network'): str, 'cloud_identifier': str, | ||
| Optional('sshPublicKeyFiles'): [str], Optional('sshTimeout'): int, | ||
| Optional('cloudScheduling'): {Optional('sshTimeout'): int}, Optional('autoMount'): bool, | ||
| Optional('masterMounts'): [str], Optional('nfsShares'): [str], | ||
| Optional('userRoles'): [{'hosts': [str], 'roles': [{'name': str, Optional('tags'): [str]}]}], | ||
| Optional('localFS'): bool, Optional('localDNSlookup'): bool, Optional('slurm'): bool, | ||
| Optional('slurmConf'): {Optional('db'): str, Optional('db_user'): str, Optional('db_password'): str, | ||
| Optional('munge_key'): str, Optional('elastic_scheduling'): {Optional('SuspendTime'): int, | ||
| Optional( | ||
| 'ResumeTimeout'): int, | ||
| Optional('TreeWidth'): int}}, | ||
| Optional('zabbix'): bool, Optional('nfs'): bool, Optional('ide'): bool, Optional('useMasterAsCompute'): bool, | ||
| Optional('useMasterWithPublicIp'): bool, Optional('waitForServices'): [str], | ||
| Optional('gateway'): {'ip': str, 'portFunction': str}, Optional('fallbackOnOtherImage'): bool, | ||
| Optional('localDNSLookup'): bool, Optional('features'): [str], 'workerInstances': [ | ||
| {'type': str, 'image': str, Optional('count'): int, Optional('onDemand'): bool, Optional('partitions'): [str], | ||
| Optional('features'): [str]}], | ||
| 'masterInstance': {'type': str, 'image': str, Optional('partitions'): [str], Optional('features'): [str]}, | ||
| Optional('vpngtw'): {'type': str, 'image': str}}) | ||
|
|
||
| other_schema = Schema( | ||
| {'infrastructure': str, 'cloud': str, 'sshUser': str, Or('subnet', 'network'): str, 'cloud_identifier': str, | ||
| Optional('waitForServices'): [str], Optional('features'): [str], 'workerInstances': [ | ||
| {'type': str, 'image': str, Optional('count'): int, Optional('onDemand'): bool, Optional('partitions'): [str], | ||
| Optional('features'): [str]}], 'vpnInstance': {'type': str, 'image': str}}) | ||
|
|
||
|
|
||
| def validate_configurations(configurations, log): | ||
| log.info("Validating config file schema...") | ||
| configuration = None | ||
| try: | ||
| configuration = configurations[0] | ||
| if configuration.get("region") or configuration.get("availabilityZone"): | ||
| log.warning( | ||
| "Keys 'region' and 'availabilityZone' are deprecated! Check will return False if you use one of them." | ||
| "Just remove them. They are no longer required.") | ||
| master_schema.validate(configuration) | ||
| log.debug(f"Master configuration '{configuration['cloud_identifier']}' valid.") | ||
| for configuration in configurations[1:]: | ||
| if configuration.get("region") or configuration.get("availabilityZone"): | ||
| log.warning( | ||
| "Keys region and availabilityZone are deprecated! Check will return False if you use one of them." | ||
| "Just remove them. They are no longer required.") | ||
| other_schema.validate(configuration) | ||
| log.debug(f"Configuration '{configuration['cloud_identifier']}' schema valid.") | ||
| log.debug("Entire configuration schema valid.") | ||
| return True | ||
| except SchemaError as err: | ||
| log.warning( | ||
| f"Configuration '{configuration.get('cloud_identifier', 'No identifier found')}' invalid. See error: " | ||
| f"{err}.") | ||
| return False |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,82 @@ | ||
| # See https://cloud.denbi.de/wiki/Tutorials/BiBiGrid/ (after update) | ||
| # First configuration will be used for general cluster information and must include the master. | ||
| # All other configurations mustn't include another master, but exactly one vpnWorker instead (keys like master). | ||
|
|
||
| - infrastructure: openstack # former mode. | ||
| cloud: some_cloud #credentials # name of clouds.yaml entry | ||
|
|
||
| sshTimeout: 6 | ||
|
|
||
| # customAnsibleCfg: True # If True, changes in ansible.cfg are kept. Default False. | ||
| # customSlurmTemplate: True # If True, changes in slurm.j2 are kept. Default False. | ||
|
|
||
| cloudScheduling: | ||
| sshTimeout: 42 | ||
| # -- BEGIN: GENERAL CLUSTER INFORMATION -- | ||
| # deleteTmpKeypairAfter: True | ||
| # dontUploadCredentials: True | ||
| ## sshPublicKeyFiles listed here will be added to access the cluster. A temporary key is created by bibigrid itself. | ||
| # - [key one] | ||
| ## Volumes and snapshots that will be mounted to master | ||
| #autoMount: True | ||
| #masterMounts: | ||
| # - test | ||
|
|
||
| #nfsShares: | ||
| # - test2 | ||
|
|
||
| ## Ansible (Galaxy) roles can be added for execution | ||
| #userRoles: | ||
| # - hosts: | ||
| # - "master" | ||
| # roles: | ||
| # - name: "resistance_nextflow" | ||
|
|
||
| ## Uncomment if you don't want assign a public ip to the master; for internal cluster (Tuebingen). | ||
| # useMasterWithPublicIp: False | ||
|
|
||
| # Other keys | ||
| #localFS: False | ||
| #localDNSlookup: False | ||
| zabbix: True | ||
| nfs: True | ||
| ide: True | ||
|
|
||
| useMasterAsCompute: True | ||
| waitForServices: | ||
| - some.service | ||
|
|
||
| fallbackOnOtherImage: True | ||
|
|
||
|
|
||
| # master configuration | ||
| # -- END: GENERAL CLUSTER INFORMATION -- | ||
|
|
||
| workerInstances: | ||
| - type: de.NBI small + ephemeral | ||
| image: ^Ubuntu 22\.04 LTS \(.*\)$ | ||
| count: 2 | ||
| #partitions: | ||
| # - ephemeral | ||
| - type: de.NBI small | ||
| image: ^Ubuntu 22\.04 LTS \(.*\)$ | ||
| count: 1 | ||
| #onDemand: False | ||
| # worker configuration | ||
|
|
||
| # Depends on cloud image | ||
| sshUser: ubuntu | ||
|
|
||
| # Depends on cloud site and project | ||
| # network: bibiserv-external | ||
| # network: bibiserv_test2_network | ||
| subnet: subnet | ||
| #gateway: | ||
| # ip: 129.70.51.103 | ||
| # portFunction: "30000 + oct4" | ||
|
|
||
| # Uncomment if no full DNS service for started instances is available. | ||
| # Currently the case in Berlin, DKFZ, Heidelberg and Tuebingen. | ||
| #localDNSLookup: yes | ||
|
|
||
| #- [next configurations] |
Oops, something went wrong.