#!/bin/sh # # Simple script for mirroring eROSITA archive # Jeremy Sanders (2023) # # Please report problems on https://erosita-forum.mpe.mpg.de/ # # BEFORE USING PLEASE MAKE SURE YOU NEED THE WHOLE ARCHIVE AND TRY TO # BE NICE TO OTHER USERS, PARTICULARLY SOON AFTER THE DATA RELEASE! # # WE INCLUDE A SPEED LIMIT BELOW - PLEASE DO NOT MODIFY UNTIL WELL # AFTER DATA RELEASE TO AVOID SERVER OVERLOAD # # You may need to uncomment the line "set ssl:verify-certificate no" # near the bottom if you get a certificate verification error, due to # a bug in some lftp versions. #### Options which can be changed below ###################################### # # directory to place archive output=ero_archive # # Choose which types of products you would like (y/n) GET_EVENT_FILES=n GET_IMAGES=n GET_EXPOSURE_MAPS=n GET_BACKGROUND_IMAGES=n GET_SENSITIVITY_MAPS=n GET_SURVEY_AREA_TABLES=n GET_SOURCE_PRODUCTS=n GET_UPPER_LIMITS=y # ############################################################################## set -e if ! command -v lftp > /dev/null; then echo "Could not find the lftp command" > /dev/stderr exit 1 fi if ! command -v wget > /dev/null ; then echo "Could not find the wget command" > /dev/stderr exit 1 fi # get list of patterns to exclude from download excludes="" if [ "$GET_EVENT_FILES" = n ]; then excludes='--exclude-glob=????_??????_???_EventList_c???.fits.gz'" $excludes" fi if [ "$GET_IMAGES" = n ]; then excludes='--exclude-glob=????_??????_???_Image_c???.fits.gz'" $excludes" fi if [ "$GET_EXPOSURE_MAPS" = n ]; then excludes='--exclude-glob=????_??????_???_ExposureMap_c???.fits.gz'" $excludes" fi if [ "$GET_BACKGROUND_IMAGES" = n ]; then excludes='--exclude-glob=????_??????_???_BackgrImage_c???.fits.gz'" $excludes" fi if [ "$GET_SENSITIVITY_MAPS" = n ]; then excludes='--exclude-glob=????_??????_???_*SensMap*_c???.fits.gz'" $excludes" fi if [ "$GET_SURVEY_AREA_TABLES" = n ]; then excludes='--exclude-glob=????_??????_???_AreaTab??????_c???.fits.gz'" $excludes" fi if [ "$GET_SOURCE_PRODUCTS" = n ]; then excludes='--exclude-glob=????_??????_???_SourceProducts_?????_c???.tar.xz'" $excludes" fi if [ "$GET_UPPER_LIMITS" = n ]; then excludes='--exclude-glob=????_??????_???_UpperLimitTab_c???.fits.gz'" $excludes" fi # get root certificate for lftp download cert=ssl-cert.pem if [ ! -f $cert ]; then echo 'Downloading certificate' wget --quiet \ -O $cert \ https://erosita.mpe.mpg.de/dr1/AllSkySurveyData_dr1/letsencrypt_r3.pem echo fi # create output directory mkdir -p $output # actually do the mirroring with lftp echo 'Doing mirroring (press Ctrl+C to stop)' lftp <