From 57798c92b5dc3d0cf7aa20f8f580cbeaa472b6bc Mon Sep 17 00:00:00 2001 From: Alexandre Date: Fri, 21 Mar 2025 07:39:30 +0100 Subject: [PATCH] use symlinks --- .../rootfs/custom-services.d/30-monitoring.sh | 294 +++++++++++++---- .../etc/cont-finish.d/savestreamdata.sh | 30 +- .../rootfs/etc/cont-init.d/01-structure.sh | 43 +-- .../etc/cont-init.d/02-restorestreamdata.sh | 34 +- .../rootfs/etc/cont-init.d/31-checks.sh | 31 +- .../rootfs/etc/cont-init.d/33-mqtt.sh | 84 +++++ .../rootfs/etc/cont-init.d/71-newfeatures.sh | 66 ++++ .../etc/cont-init.d/81-modifications.sh | 112 +++++-- .../etc/cont-init.d/91-nginx_ingress.sh | 30 +- .../rootfs/etc/cont-init.d/92-ssl.sh | 11 + .../rootfs/etc/cont-init.d/98-oldcpu.sh | 37 +++ .../rootfs/etc/cont-init.d/99-run.sh | 33 +- .../rootfs/etc/nginx/servers/ingress.conf | 28 +- .../rootfs/helpers/birdnet_to_mqtt.py | 6 +- .../rootfs/helpers/convert_list.php | 116 ------- battybirdnet-pi/rootfs/helpers/journalctl3.py | 31 ++ battybirdnet-pi/rootfs/helpers/systemctl3.py | 300 +++++++++--------- battybirdnet-pi/rootfs/helpers/timedatectl | 2 +- battybirdnet-pi/rootfs/helpers/views.add | 27 -- 19 files changed, 850 insertions(+), 465 deletions(-) create mode 100644 battybirdnet-pi/rootfs/etc/cont-init.d/33-mqtt.sh create mode 100644 battybirdnet-pi/rootfs/etc/cont-init.d/71-newfeatures.sh create mode 100644 battybirdnet-pi/rootfs/etc/cont-init.d/98-oldcpu.sh delete mode 100644 battybirdnet-pi/rootfs/helpers/convert_list.php create mode 100644 battybirdnet-pi/rootfs/helpers/journalctl3.py delete mode 100644 battybirdnet-pi/rootfs/helpers/views.add diff --git a/battybirdnet-pi/rootfs/custom-services.d/30-monitoring.sh b/battybirdnet-pi/rootfs/custom-services.d/30-monitoring.sh index ff3e9fbcc..3bfa9185b 100755 --- a/battybirdnet-pi/rootfs/custom-services.d/30-monitoring.sh +++ b/battybirdnet-pi/rootfs/custom-services.d/30-monitoring.sh @@ -1,105 +1,261 @@ #!/usr/bin/env bash # shellcheck shell=bash -# Adapted from https://github.com/mcguirepr89/BirdNET-Pi/issues/393#issuecomment-1166445710 +# Improved BirdNET-Pi Monitoring Script with Recovery Alerts and Condensed Logs HOME="/home/pi" -# Define logging functions -log_green() { echo -e "\033[32m$1\033[0m"; } -log_red() { echo -e "\033[31m$1\033[0m"; } -log_yellow() { echo -e "\033[33m$1\033[0m"; } -log_info() { echo -e "\033[34m$1\033[0m"; } - -echo "$(log_green "Starting service: throttlerecording")" -touch "$HOME/BirdSongs/StreamData/analyzing_now.txt" +######################################## +# Logging Functions (color-coded for terminal clarity) +######################################## +log_green() { echo -e "\033[32m$1\033[0m"; } +log_red() { echo -e "\033[31m$1\033[0m"; } +log_yellow() { echo -e "\033[33m$1\033[0m"; } +log_blue() { echo -e "\033[34m$1\033[0m"; } +######################################## # Read configuration +######################################## set +u +# shellcheck disable=SC1091 source /etc/birdnet/birdnet.conf -# Set constants -srv="birdnet_recording" -srv2="birdnet_analysis" -ingest_dir="$RECS_DIR/StreamData" -counter=10 -# Ensure directories and permissions -mkdir -p "$ingest_dir" -chown -R pi:pi "$ingest_dir" -chmod -R 755 "$ingest_dir" +######################################## +# Wait 5 minutes for system stabilization +######################################## +sleep 5m -# Function to send notifications using Apprise +log_green "Starting service: throttlerecording" + +######################################## +# Define Directories, Files, and Constants +######################################## +INGEST_DIR="$(readlink -f "$HOME/BirdSongs/StreamData")" +ANALYZING_NOW_FILE="$INGEST_DIR/analyzing_now.txt" +touch "$ANALYZING_NOW_FILE" +BIRDSONGS_DIR="$(readlink -f "$HOME/BirdSongs/Extracted/By_Date")" + +# Ensure directories and set permissions +mkdir -p "$INGEST_DIR" || { log_red "Failed to create directory: $INGEST_DIR"; exit 1; } +chown -R pi:pi "$INGEST_DIR" || log_yellow "Could not change ownership for $INGEST_DIR" +chmod -R 755 "$INGEST_DIR" || log_yellow "Could not set permissions for $INGEST_DIR" + +# Services to monitor +SERVICES=(birdnet_analysis chart_viewer spectrogram_viewer birdnet_recording birdnet_log birdnet_stats) + +######################################## +# Notification settings +######################################## +NOTIFICATION_INTERVAL=1800 # 30 minutes in seconds +NOTIFICATION_INTERVAL_IN_MINUTES=$(( NOTIFICATION_INTERVAL / 60 )) +last_notification_time=0 +issue_reported=0 # 1 = an issue was reported, 0 = system is normal +declare -A SERVICE_INACTIVE_COUNT=() + +# Disk usage threshold (percentage) +DISK_USAGE_THRESHOLD=95 + +# "Analyzing" file check variables +same_file_counter=0 +SAME_FILE_THRESHOLD=2 +if [[ -f "$ANALYZING_NOW_FILE" ]]; then + analyzing_now=$(<"$ANALYZING_NOW_FILE") +else + analyzing_now="" +fi + +######################################## +# Notification Functions +######################################## apprisealert() { - local notification="" - local stopped_service="
Stopped services: " + local issue_message="$1" + local current_time + current_time=$(date +%s) - # Check for stopped services - services=(birdnet_analysis chart_viewer spectrogram_viewer icecast2 birdnet_recording birdnet_log birdnet_stats) - for service in "${services[@]}"; do - if [[ "$(systemctl is-active "$service")" == "inactive" ]]; then + # Calculate time_diff in minutes since last notification + local time_diff=$(( (current_time - last_notification_time) / 60 )) + + # Throttle notifications + if (( time_diff < NOTIFICATION_INTERVAL_IN_MINUTES )); then + log_yellow "Notification suppressed (last sent ${time_diff} minutes ago)." + return + fi + + local stopped_service="
Stopped services: " + for service in "${SERVICES[@]}"; do + if [[ "$(systemctl is-active "$service")" != "active" ]]; then stopped_service+="$service; " fi done - # Build notification message + local notification="Issue: $issue_message" notification+="$stopped_service" - notification+="
Additional information: " - notification+="
Since: ${LASTCHECK:-unknown}" notification+="
System: ${SITE_NAME:-$(hostname)}" - notification+="
Available disk space: $(df -h "$HOME/BirdSongs" | awk 'NR==2 {print $4}')" + notification+="
Available disk space: $(df -h "$BIRDSONGS_DIR" | awk 'NR==2 {print $4}')" + notification+="
----Last log lines----" + notification+="
$(timeout 15 cat /proc/1/fd/1 | head -n 5)" + notification+="
----------------------" [[ -n "$BIRDNETPI_URL" ]] && notification+="
Access your BirdNET-Pi" - # Send notification - TITLE="BirdNET-Analyzer stopped" - "$HOME/BirdNET-Pi/birdnet/bin/apprise" -vv -t "$TITLE" -b "$notification" --input-format=html --config="$HOME/BirdNET-Pi/apprise.txt" + local TITLE="BirdNET-Analyzer Alert" + if [[ -f "$HOME/BirdNET-Pi/birdnet/bin/apprise" && -s "$HOME/BirdNET-Pi/apprise.txt" ]]; then + "$HOME/BirdNET-Pi/birdnet/bin/apprise" -vv -t "$TITLE" -b "$notification" \ + --input-format=html --config="$HOME/BirdNET-Pi/apprise.txt" + last_notification_time=$current_time + issue_reported=1 + else + log_red "Apprise not configured or missing!" + fi } -# Main loop -while true; do - sleep 61 +apprisealert_recovery() { + # Only send a recovery message if we had previously reported an issue + if (( issue_reported == 1 )); then + log_green "$(date) INFO: System is back to normal. Sending recovery notification." - # Restart analysis if clogged - if ((counter <= 0)); then - current_file="$(cat "$ingest_dir/analyzing_now.txt")" - if [[ "$current_file" == "$analyzing_now" ]]; then - log_yellow "$(date) WARNING: no change in analyzing_now for 10 iterations, restarting services" - "$HOME/BirdNET-Pi/scripts/restart_services.sh" + local TITLE="BirdNET-Pi System Recovered" + local notification="All monitored services are back to normal.
" + notification+="System: ${SITE_NAME:-$(hostname)}
" + notification+="Available disk space: $(df -h "$BIRDSONGS_DIR" | awk 'NR==2 {print $4}')" + + if [[ -f "$HOME/BirdNET-Pi/birdnet/bin/apprise" && -s "$HOME/BirdNET-Pi/apprise.txt" ]]; then + "$HOME/BirdNET-Pi/birdnet/bin/apprise" -vv -t "$TITLE" -b "$notification" \ + --input-format=html --config="$HOME/BirdNET-Pi/apprise.txt" fi - counter=10 + issue_reported=0 + fi +} + +######################################## +# Helper Checks +######################################## + +check_disk_space() { + local current_usage + current_usage=$(df -h "$BIRDSONGS_DIR" | awk 'NR==2 {print $5}' | sed 's/%//') + + if (( current_usage >= DISK_USAGE_THRESHOLD )); then + log_red "$(date) INFO: Disk usage is at ${current_usage}% (CRITICAL!)" + apprisealert "Disk usage critical: ${current_usage}%" + return 1 + else + log_green "$(date) INFO: Disk usage is within acceptable limits (${current_usage}%)." + return 0 + fi +} + +check_analyzing_now() { + local current_file + current_file=$(cat "$ANALYZING_NOW_FILE" 2>/dev/null) + if [[ "$current_file" == "$analyzing_now" ]]; then + (( same_file_counter++ )) + else + same_file_counter=0 analyzing_now="$current_file" fi - # Check recorder state and queue length - wav_count=$(find "$ingest_dir" -maxdepth 1 -name '*.wav' | wc -l) - service_state=$(systemctl is-active "$srv") - analysis_state=$(systemctl is-active "$srv2") - - log_green "$(date) INFO: $wav_count wav files waiting in $ingest_dir, $srv state is $service_state, $srv2 state is $analysis_state" - - # Pause recorder if queue is too large - if ((wav_count > 50)); then - log_red "$(date) WARNING: Too many files in queue, pausing $srv and restarting $srv2" - sudo systemctl stop "$srv" - sudo systemctl restart "$srv2" - sleep 30 - elif ((wav_count > 30)); then - log_red "$(date) WARNING: Too many files in queue, restarting $srv2" - sudo systemctl restart "$srv2" - sleep 30 + if (( same_file_counter >= SAME_FILE_THRESHOLD )); then + log_red "$(date) INFO: 'analyzing_now' file unchanged for $SAME_FILE_THRESHOLD iterations." + apprisealert "No change in analyzing_now for ${SAME_FILE_THRESHOLD} iterations" + "$HOME/BirdNET-Pi/scripts/restart_services.sh" + same_file_counter=0 + return 1 + else + # Only log if it changed this iteration + if (( same_file_counter == 0 )); then + log_green "$(date) INFO: 'analyzing_now' file has been updated." + fi + return 0 fi +} - # Check service states - for service in "$srv" "$srv2"; do - state_var="${service}_state" - if [[ "${state_var:-}" != "active" ]]; then - log_yellow "$(date) INFO: Restarting $service service" - sudo systemctl restart "$service" +check_queue() { + local wav_count + wav_count=$(find -L "$INGEST_DIR" -maxdepth 1 -name '*.wav' | wc -l) + + log_green "$(date) INFO: Queue is at a manageable level (${wav_count} wav files)." + + if (( wav_count > 50 )); then + log_red "$(date) INFO: Queue >50. Stopping recorder + restarting analyzer." + apprisealert "Queue exceeded 50: stopping recorder, restarting analyzer." + sudo systemctl stop birdnet_recording + sudo systemctl restart birdnet_analysis + return 1 + elif (( wav_count > 30 )); then + log_red "$(date) INFO: Queue >30. Restarting analyzer." + apprisealert "Queue exceeded 30: restarting analyzer." + sudo systemctl restart birdnet_analysis + return 1 + fi + return 0 +} + +check_services() { + local any_inactive=0 + + for service in "${SERVICES[@]}"; do + if [[ "$(systemctl is-active "$service")" != "active" ]]; then + SERVICE_INACTIVE_COUNT["$service"]=$(( SERVICE_INACTIVE_COUNT["$service"] + 1 )) + + if (( SERVICE_INACTIVE_COUNT["$service"] == 1 )); then + # First time inactive => Try to start + log_yellow "$(date) INFO: Service '$service' is inactive. Attempting to start..." + systemctl start "$service" + any_inactive=1 + elif (( SERVICE_INACTIVE_COUNT["$service"] == 2 )); then + # Second consecutive time => Send an alert + log_red "$(date) INFO: Service '$service' is still inactive after restart attempt." + apprisealert "Service '$service' remains inactive after restart attempt." + any_inactive=1 + else + # Beyond second check => keep logging or do advanced actions + log_red "$(date) INFO: Service '$service' inactive for ${SERVICE_INACTIVE_COUNT["$service"]} checks in a row." + any_inactive=1 + fi + else + # Service is active => reset counter + if (( SERVICE_INACTIVE_COUNT["$service"] > 0 )); then + log_green "$(date) INFO: Service '$service' is back to active. Resetting counter." + fi + SERVICE_INACTIVE_COUNT["$service"]=0 fi done - # Send alert if needed - if ((wav_count > 30)) && [[ -s "$HOME/BirdNET-Pi/apprise.txt" ]]; then - apprisealert + if (( any_inactive == 0 )); then + log_green "$(date) INFO: All services are active" + return 0 + else + log_red "$(date) INFO: One or more services are inactive" + return 1 fi +} - ((counter--)) +######################################## +# Main Monitoring Loop +######################################## +while true; do + sleep 61 + log_blue "----------------------------------------" + log_blue "$(date) INFO: Starting monitoring check" + any_issue=0 + + # 1) Disk usage + check_disk_space || any_issue=1 + + # 2) 'analyzing_now' file + check_analyzing_now || any_issue=1 + + # 3) Queue check + check_queue || any_issue=1 + + # 4) Services check + check_services || any_issue=1 + + # Final summary + if (( any_issue == 0 )); then + log_green "$(date) INFO: All systems are functioning normally" + apprisealert_recovery + else + log_red "$(date) INFO: Issues detected. System status is not fully operational." + fi + log_blue "----------------------------------------" done diff --git a/battybirdnet-pi/rootfs/etc/cont-finish.d/savestreamdata.sh b/battybirdnet-pi/rootfs/etc/cont-finish.d/savestreamdata.sh index de9eaae80..662817120 100755 --- a/battybirdnet-pi/rootfs/etc/cont-finish.d/savestreamdata.sh +++ b/battybirdnet-pi/rootfs/etc/cont-finish.d/savestreamdata.sh @@ -1,6 +1,16 @@ #!/usr/bin/with-contenv bashio # shellcheck shell=bash +# Maximum file size in bytes (50MB) +MAX_SIZE=$((50 * 1024 * 1024)) + +# Function to check if a file is a valid WAV +is_valid_wav() { + local file="$1" + # Check if the file contains a valid WAV header + file "$file" | grep -qE 'WAVE audio' +} + if [ -d "$HOME"/BirdSongs/StreamData ]; then bashio::log.fatal "Container stopping, saving temporary files." @@ -18,16 +28,22 @@ if [ -d "$HOME"/BirdSongs/StreamData ]; then # Wait for both services to stop wait - # Check if there are files in StreamData and move them to /data/StreamData + # Create the destination directory mkdir -p /data/StreamData - if [ "$(ls -A "$HOME"/BirdSongs/StreamData)" ]; then - if mv -v "$HOME"/BirdSongs/StreamData/* /data/StreamData/; then - bashio::log.info "Files successfully moved to /data/StreamData." + + # Move only valid WAV files under 50MB + shopt -s nullglob # Prevent errors if no files match + for file in "$HOME"/BirdSongs/StreamData/*.wav; do + if [ -f "$file" ] && [ "$(stat --format="%s" "$file")" -lt "$MAX_SIZE" ] && is_valid_wav "$file"; then + if mv -v "$file" /data/StreamData/; then + bashio::log.info "Moved valid WAV file: $(basename "$file")" + else + bashio::log.error "Failed to move: $(basename "$file")" + fi else - bashio::log.error "Failed to move files to /data/StreamData." - exit 1 + bashio::log.warning "Skipping invalid or large file: $(basename "$file")" fi - fi + done bashio::log.info "... files safe, allowing container to stop." else diff --git a/battybirdnet-pi/rootfs/etc/cont-init.d/01-structure.sh b/battybirdnet-pi/rootfs/etc/cont-init.d/01-structure.sh index 81d410038..549f4780c 100755 --- a/battybirdnet-pi/rootfs/etc/cont-init.d/01-structure.sh +++ b/battybirdnet-pi/rootfs/etc/cont-init.d/01-structure.sh @@ -2,6 +2,17 @@ # shellcheck shell=bash set -e +################## +# ALLOW RESTARTS # +################## + +if [[ "${BASH_SOURCE[0]}" == /etc/cont-init.d/* ]]; then + mkdir -p /etc/scripts-init + sed -i "s|/etc/cont-init.d|/etc/scripts-init|g" /ha_entrypoint.sh + sed -i "/ rm/d" /ha_entrypoint.sh + cp "${BASH_SOURCE[0]}" /etc/scripts-init/ +fi + ############### # SET /CONFIG # ############### @@ -18,8 +29,6 @@ for file in "${DEFAULT_FILES[@]}"; do done touch /config/include_species_list.txt # Ensure this is always created -touch "$HOME/BirdNET-Pi/scripts/common.php" - # Set BirdSongs folder location from configuration if specified BIRDSONGS_FOLDER="/config/BirdSongs" if bashio::config.has_value "BIRDSONGS_FOLDER"; then @@ -53,6 +62,14 @@ echo "... setting permissions for user pi" chown -R pi:pi /config /etc/birdnet "$BIRDSONGS_FOLDER" /tmp chmod -R 755 /config /etc/birdnet "$BIRDSONGS_FOLDER" /tmp +# Backup default birdnet.conf for sanity check +cp "$HOME/BirdNET-Pi/birdnet.conf" "$HOME/BirdNET-Pi/birdnet.bak" + +# Create default birdnet.conf if not existing +if [ ! -f /config/birdnet.conf ]; then + cp -f "$HOME/BirdNET-Pi/birdnet.conf" /config/ +fi + # Create default birds.db if [ ! -f /config/birds.db ]; then echo "... creating initial db" @@ -65,12 +82,6 @@ elif [ "$(stat -c%s /config/birds.db)" -lt 10240 ]; then cp "$HOME/BirdNET-Pi/scripts/birds.db" /config/ fi -# Backup default birdnet.conf for sanity check -if [ ! -f "$HOME/BirdNET-Pi/birdnet.conf" ]; then - ln -s /etc/birdnet/birdnet.conf "$HOME/BirdNET-Pi/birdnet.conf" -fi -cp "$HOME/BirdNET-Pi/birdnet.conf" "$HOME/BirdNET-Pi/birdnet.bak" - # Symlink configuration files echo "... creating symlinks for configuration files" CONFIG_FILES=("$HOME/BirdNET-Pi/birdnet.conf" "$HOME/BirdNET-Pi/scripts/whitelist_species_list.txt" "$HOME/BirdNET-Pi/blacklisted_images.txt" "$HOME/BirdNET-Pi/scripts/birds.db" "$HOME/BirdNET-Pi/BirdDB.txt" "$HOME/BirdNET-Pi/scripts/disk_check_exclude.txt" "$HOME/BirdNET-Pi/apprise.txt" "$HOME/BirdNET-Pi/exclude_species_list.txt" "$HOME/BirdNET-Pi/include_species_list.txt" "$HOME/BirdNET-Pi/IdentifiedSoFar.txt" "$HOME/BirdNET-Pi/scripts/confirmed_species_list.txt") @@ -84,12 +95,6 @@ for file in "${CONFIG_FILES[@]}"; do sudo -u pi ln -fs "/config/$filename" "/etc/birdnet/$filename" done -# thisrun -cp /config/birdnet.conf "$HOME/BirdNET-Pi/scripts/thisrun.txt" -cp /config/birdnet.conf "$HOME/BirdNET-Pi/scripts/lastrun.txt" -chown pi:pi "$HOME/BirdNET-Pi/scripts/thisrun.txt" -chown pi:pi "$HOME/BirdNET-Pi/scripts/lastrun.txt" - # Symlink BirdSongs folders for folder in By_Date Charts; do echo "... creating symlink for $BIRDSONGS_FOLDER/$folder" @@ -102,9 +107,9 @@ echo "... checking and setting permissions" chmod -R 755 /config/* chmod 777 /config -# Create Matplotlib configuration directory +# Create folder for matplotlib echo "... setting up Matplotlabdir" -MPLCONFIGDIR="${MPLCONFIGDIR:-$HOME/.config/matplotlib}" -mkdir -p "$MPLCONFIGDIR" -chown pi:pi "$MPLCONFIGDIR" -chmod 777 "$MPLCONFIGDIR" +mkdir -p "$HOME"/.cache/matplotlib +chown -R "pi:pi" "$HOME"/.cache/matplotlib +chmod 777 "$HOME"/.cache/matplotlib + diff --git a/battybirdnet-pi/rootfs/etc/cont-init.d/02-restorestreamdata.sh b/battybirdnet-pi/rootfs/etc/cont-init.d/02-restorestreamdata.sh index 8ab4ae674..e51a2f8dc 100755 --- a/battybirdnet-pi/rootfs/etc/cont-init.d/02-restorestreamdata.sh +++ b/battybirdnet-pi/rootfs/etc/cont-init.d/02-restorestreamdata.sh @@ -2,22 +2,38 @@ # shellcheck shell=bash set -e -if [ -d /data/StreamData ]; then - bashio::log.fatal "Container was stopped while files were still being analyzed." +################## +# ALLOW RESTARTS # +################## - # Check if there are .wav files in /data/StreamData - if find /data/StreamData -type f -name "*.wav" | grep -q .; then - bashio::log.warning "Restoring .wav files from /data/StreamData to $HOME/BirdSongs/StreamData." +if [[ "${BASH_SOURCE[0]}" == /etc/cont-init.d/* ]]; then + mkdir -p /etc/scripts-init + sed -i "s|/etc/cont-init.d|/etc/scripts-init|g" /ha_entrypoint.sh + sed -i "/ rm/d" /ha_entrypoint.sh + cp "${BASH_SOURCE[0]}" /etc/scripts-init/ +fi + +###################### +# RESTORE STREAMDATA # +###################### + +if [ -d /config/TemporaryFiles ]; then + + # Check if there are .wav files in /config/TemporaryFiles + if find /config/TemporaryFiles -type f -name "*.wav" | grep -q .; then + bashio::log.warning "Container was stopped while files were still being analyzed." + echo "... restoring .wav files from /config/TemporaryFiles to $HOME/BirdSongs/StreamData." # Create the destination directory if it does not exist mkdir -p "$HOME"/BirdSongs/StreamData # Count the number of .wav files to be moved - file_count=$(find /data/StreamData -type f -name "*.wav" | wc -l) + file_count=$(find /config/TemporaryFiles -type f -name "*.wav" | wc -l) echo "... found $file_count .wav files to restore." # Move the .wav files using `mv` to avoid double log entries - mv -v /data/StreamData/*.wav "$HOME"/BirdSongs/StreamData/ + mv -v /config/TemporaryFiles/*.wav "$HOME"/BirdSongs/StreamData/ + rm -r /config/TemporaryFiles # Update permissions only if files were moved successfully if [ "$file_count" -gt 0 ]; then @@ -30,7 +46,7 @@ if [ -d /data/StreamData ]; then fi # Clean up the source folder if it is empty - if [ -z "$(ls -A /data/StreamData)" ]; then - rm -r /data/StreamData + if [ -z "$(ls -A /config/TemporaryFiles)" ]; then + rm -r /config/TemporaryFiles fi fi diff --git a/battybirdnet-pi/rootfs/etc/cont-init.d/31-checks.sh b/battybirdnet-pi/rootfs/etc/cont-init.d/31-checks.sh index c33a2d470..b76d3ded3 100755 --- a/battybirdnet-pi/rootfs/etc/cont-init.d/31-checks.sh +++ b/battybirdnet-pi/rootfs/etc/cont-init.d/31-checks.sh @@ -2,6 +2,17 @@ # shellcheck shell=bash set -e +################## +# ALLOW RESTARTS # +################## + +if [[ "${BASH_SOURCE[0]}" == /etc/cont-init.d/* ]]; then + mkdir -p /etc/scripts-init + sed -i "s|/etc/cont-init.d|/etc/scripts-init|g" /ha_entrypoint.sh + sed -i "/ rm/d" /ha_entrypoint.sh + cp "${BASH_SOURCE[0]}" /etc/scripts-init/ +fi + ###################### # CHECK BIRDNET.CONF # ###################### @@ -48,17 +59,17 @@ fi bashio::log.info "Performing potential updates" # Adapt update_birdnet_snippets -sed -i "/USER=/a USER=\"$USER\"" "$HOME"/BirdNET-Pi/scripts/update_birdnet_snippets.sh -sed -i "/HOME=/a HOME=\"$HOME\"" "$HOME"/BirdNET-Pi/scripts/update_birdnet_snippets.sh -sed -i "/chown/d" "$HOME"/BirdNET-Pi/scripts/update_birdnet_snippets.sh -sed -i "/chmod/d" "$HOME"/BirdNET-Pi/scripts/update_birdnet_snippets.sh +sed -i "s|systemctl list-unit-files|false \&\& echo|g" "$HOME"/BirdNET-Pi/scripts/update_birdnet_snippets.sh # Avoid systemctl +sed -i "/systemctl /d" "$HOME"/BirdNET-Pi/scripts/update_birdnet_snippets.sh # Avoid systemctl +sed -i "/install_tmp_mount/d" "$HOME"/BirdNET-Pi/scripts/update_birdnet_snippets.sh # Use HA tmp +sed -i "/find /d" "$HOME"/BirdNET-Pi/scripts/update_birdnet_snippets.sh # Not useful +sed -i "/set -x/d" "$HOME"/BirdNET-Pi/scripts/update_birdnet_snippets.sh # Not useful +sed -i "/restart_services/d" "$HOME"/BirdNET-Pi/scripts/update_birdnet_snippets.sh # Not useful sed -i "s|/etc/birdnet/birdnet.conf|/config/birdnet.conf|g" "$HOME"/BirdNET-Pi/scripts/update_birdnet_snippets.sh -sed -i "/restart_services/d" "$HOME"/BirdNET-Pi/scripts/update_birdnet_snippets.sh -sed -i "/set -x/d" "$HOME"/BirdNET-Pi/scripts/update_birdnet_snippets.sh -sed -i "s|systemctl list-unit-files|false \&\& echo|g" "$HOME"/BirdNET-Pi/scripts/update_birdnet_snippets.sh -sed -i "/systemctl /d" "$HOME"/BirdNET-Pi/scripts/update_birdnet_snippets.sh -sed -i "/find /d" "$HOME"/BirdNET-Pi/scripts/update_birdnet_snippets.sh +sed -i "/update_caddyfile/c echo \"yes\"" "$HOME"/BirdNET-Pi/scripts/update_birdnet_snippets.sh # Avoid systemctl # Execute update_birdnet_snippets +export RECS_DIR="$HOME/BirdSongs" +export EXTRACTED="$HOME/BirdSongs/Extracted" +chmod +x "$HOME"/BirdNET-Pi/scripts/update_birdnet_snippets.sh "$HOME"/BirdNET-Pi/scripts/update_birdnet_snippets.sh - diff --git a/battybirdnet-pi/rootfs/etc/cont-init.d/33-mqtt.sh b/battybirdnet-pi/rootfs/etc/cont-init.d/33-mqtt.sh new file mode 100644 index 000000000..c83248bf7 --- /dev/null +++ b/battybirdnet-pi/rootfs/etc/cont-init.d/33-mqtt.sh @@ -0,0 +1,84 @@ +#!/command/with-contenv bashio +# shellcheck shell=bash +set -e + +################## +# ALLOW RESTARTS # +################## + +if [[ "${BASH_SOURCE[0]}" == /etc/cont-init.d/* ]]; then + mkdir -p /etc/scripts-init + sed -i "s|/etc/cont-init.d|/etc/scripts-init|g" /ha_entrypoint.sh + sed -i "/ rm/d" /ha_entrypoint.sh + cp "${BASH_SOURCE[0]}" /etc/scripts-init/ +fi + +############ +# SET MQTT # +############ + +# Function to perform common setup steps +common_steps () { + # Attempt to connect to the MQTT broker + TOPIC="birdnet" + if mosquitto_pub -h "$MQTT_HOST" -p "$MQTT_PORT" -t "$TOPIC" -m "test" -u "$MQTT_USER" -P "$MQTT_PASS" -q 1 -d --will-topic "$TOPIC" --will-payload "Disconnected" --will-qos 1 --will-retain > /dev/null 2>&1; then + # Adapt script with MQTT settings + sed -i "s|%%mqtt_server%%|$MQTT_HOST|g" /helpers/birdnet_to_mqtt.py + sed -i "s|%%mqtt_port%%|$MQTT_PORT|g" /helpers/birdnet_to_mqtt.py + sed -i "s|%%mqtt_user%%|$MQTT_USER|g" /helpers/birdnet_to_mqtt.py + sed -i "s|%%mqtt_pass%%|$MQTT_PASS|g" /helpers/birdnet_to_mqtt.py + + # Copy script to the appropriate directory + cp /helpers/birdnet_to_mqtt.py "$HOME"/BirdNET-Pi/scripts/utils/birdnet_to_mqtt.py + chown pi:pi "$HOME"/BirdNET-Pi/scripts/utils/birdnet_to_mqtt.py + chmod +x "$HOME"/BirdNET-Pi/scripts/utils/birdnet_to_mqtt.py + + # Add hooks to the main analysis script + sed -i "/load_global_model, run_analysis/a from utils.birdnet_to_mqtt import automatic_mqtt_publish" "$HOME"/BirdNET-Pi/scripts/birdnet_analysis.py + sed -i '/write_to_db(/a\ automatic_mqtt_publish(file, detection, os.path.basename(detection.file_name_extr))' "$HOME"/BirdNET-Pi/scripts/birdnet_analysis.py + else + bashio::log.fatal "MQTT connection failed, it will not be configured" + fi +} + +# Check if MQTT service is available and not disabled +if bashio::services.available 'mqtt' && ! bashio::config.true 'MQTT_DISABLED'; then + bashio::log.green "---" + bashio::log.blue "MQTT addon is active on your system! Birdnet-pi is now automatically configured to send its output to MQTT" + bashio::log.blue "MQTT user : $(bashio::services "mqtt" "username")" + bashio::log.blue "MQTT password : $(bashio::services "mqtt" "password")" + bashio::log.blue "MQTT broker : tcp://$(bashio::services "mqtt" "host"):$(bashio::services "mqtt" "port")" + bashio::log.green "---" + bashio::log.blue "Data will be posted to the topic : 'birdnet'" + bashio::log.blue "Json data : {'Date', 'Time', 'ScientificName', 'CommonName', 'Confidence', 'SpeciesCode', 'ClipName', 'url'}" + bashio::log.blue "---" + + # Apply MQTT settings + MQTT_HOST="$(bashio::services "mqtt" "host")" + MQTT_PORT="$(bashio::services "mqtt" "port")" + MQTT_USER="$(bashio::services "mqtt" "username")" + MQTT_PASS="$(bashio::services "mqtt" "password")" + + # Perform common setup steps + common_steps + +# Check if manual MQTT configuration is provided +elif bashio::config.has_value "MQTT_HOST_manual" && bashio::config.has_value "MQTT_PORT_manual"; then + bashio::log.green "---" + bashio::log.blue "MQTT is manually configured in the addon options" + bashio::log.blue "Birdnet-pi is now automatically configured to send its output to MQTT" + bashio::log.green "---" + bashio::log.blue "Data will be posted to the topic : 'birdnet'" + bashio::log.blue "Json data : {'Date', 'Time', 'ScientificName', 'CommonName', 'Confidence', 'SpeciesCode', 'ClipName', 'url'}" + bashio::log.blue "---" + + # Apply manual MQTT settings + MQTT_HOST="$(bashio::config "MQTT_HOST_manual")" + MQTT_PORT="$(bashio::config "MQTT_PORT_manual")" + MQTT_USER="$(bashio::config "MQTT_USER_manual")" + MQTT_PASS="$(bashio::config "MQTT_PASSWORD_manual")" + + # Perform common setup steps + common_steps + +fi diff --git a/battybirdnet-pi/rootfs/etc/cont-init.d/71-newfeatures.sh b/battybirdnet-pi/rootfs/etc/cont-init.d/71-newfeatures.sh new file mode 100644 index 000000000..2523d4c3e --- /dev/null +++ b/battybirdnet-pi/rootfs/etc/cont-init.d/71-newfeatures.sh @@ -0,0 +1,66 @@ +#!/command/with-contenv bashio +# shellcheck shell=bash +set -e + +################## +# ALLOW RESTARTS # +################## + +if [[ "${BASH_SOURCE[0]}" == /etc/cont-init.d/* ]]; then + mkdir -p /etc/scripts-init + sed -i "s|/etc/cont-init.d|/etc/scripts-init|g" /ha_entrypoint.sh + sed -i "/ rm/d" /ha_entrypoint.sh + cp "${BASH_SOURCE[0]}" /etc/scripts-init/ +fi + +################ +# ADD FEATURES # +################ + +bashio::log.info "Adding optional features" + +# Enable the Processed folder +############################# + +if bashio::config.true "PROCESSED_FOLDER_ENABLED" && ! grep -q "processed_size" "$HOME"/BirdNET-Pi/scripts/birdnet_analysis.py; then + echo "... Enabling the Processed folder : the last 15 wav files will be stored there" + # Adapt config.php + sed -i "/GET\[\"info_site\"\]/a\ \$processed_size = \$_GET\[\"processed_size\"\];" "$HOME"/BirdNET-Pi/scripts/config.php + sed -i "/\$contents = file_get_contents/a\ \$contents = preg_replace\(\"/PROCESSED_SIZE=\.\*/\", \"PROCESSED_SIZE=\$processed_size\", \$contents\);" "$HOME"/BirdNET-Pi/scripts/config.php + sed -i "/\"success\"/i
" "$HOME"/BirdNET-Pi/scripts/config.php + sed -i "/\"success\"/i

Processed folder management

" "$HOME"/BirdNET-Pi/scripts/config.php + sed -i "/\"success\"/i " "$HOME"/BirdNET-Pi/scripts/config.php + sed -i "/\"success\"/i \"/>" "$HOME"/BirdNET-Pi/scripts/config.php + sed -i "/\"success\"/i
" "$HOME"/BirdNET-Pi/scripts/config.php + sed -i "/\"success\"/i Processed is the directory where the formerly 'Analyzed' files are moved after extractions, mostly for troubleshooting purposes.
" "$HOME"/BirdNET-Pi/scripts/config.php + sed -i "/\"success\"/i This value defines the maximum amount of files that are kept before replacement with new files.
" "$HOME"/BirdNET-Pi/scripts/config.php + sed -i "/\"success\"/i
" "$HOME"/BirdNET-Pi/scripts/config.php + sed -i "/\"success\"/i\
" "$HOME"/BirdNET-Pi/scripts/config.php + # Adapt birdnet_analysis.py - move_to_processed + sed -i "/log.info('handle_reporting_queue done')/a\ os.remove(files.pop(0))" "$HOME"/BirdNET-Pi/scripts/birdnet_analysis.py + sed -i "/log.info('handle_reporting_queue done')/a\ while len(files) > processed_size:" "$HOME"/BirdNET-Pi/scripts/birdnet_analysis.py + sed -i "/log.info('handle_reporting_queue done')/a\ files.sort(key=os.path.getmtime)" "$HOME"/BirdNET-Pi/scripts/birdnet_analysis.py + sed -i "/log.info('handle_reporting_queue done')/a\ files = glob.glob(os.path.join(processed_dir, '*'))" "$HOME"/BirdNET-Pi/scripts/birdnet_analysis.py + sed -i "/log.info('handle_reporting_queue done')/a\ os.rename(file_name, os.path.join(processed_dir, os.path.basename(file_name)))" "$HOME"/BirdNET-Pi/scripts/birdnet_analysis.py + sed -i "/log.info('handle_reporting_queue done')/a\ processed_dir = os.path.join(get_settings()['RECS_DIR'], 'Processed')" "$HOME"/BirdNET-Pi/scripts/birdnet_analysis.py + sed -i "/log.info('handle_reporting_queue done')/a\def move_to_processed(file_name, processed_size):" "$HOME"/BirdNET-Pi/scripts/birdnet_analysis.py + sed -i "/log.info('handle_reporting_queue done')/a\ " "$HOME"/BirdNET-Pi/scripts/birdnet_analysis.py + # Adapt birdnet_analysis.py - get_processed_size + sed -i "/log.info('handle_reporting_queue done')/a\ return 0" "$HOME"/BirdNET-Pi/scripts/birdnet_analysis.py + sed -i "/log.info('handle_reporting_queue done')/a\ except (ValueError, TypeError):" "$HOME"/BirdNET-Pi/scripts/birdnet_analysis.py + sed -i "/log.info('handle_reporting_queue done')/a\ return processed_size if isinstance(processed_size, int) else 0" "$HOME"/BirdNET-Pi/scripts/birdnet_analysis.py + sed -i "/log.info('handle_reporting_queue done')/a\ processed_size = get_settings().getint('PROCESSED_SIZE')" "$HOME"/BirdNET-Pi/scripts/birdnet_analysis.py + sed -i "/log.info('handle_reporting_queue done')/a\ try:" "$HOME"/BirdNET-Pi/scripts/birdnet_analysis.py + sed -i "/log.info('handle_reporting_queue done')/a\def get_processed_size():" "$HOME"/BirdNET-Pi/scripts/birdnet_analysis.py + sed -i "/log.info('handle_reporting_queue done')/a\ " "$HOME"/BirdNET-Pi/scripts/birdnet_analysis.py + # Modify calls + sed -i "/from subprocess import CalledProcessError/a\import glob" "$HOME"/BirdNET-Pi/scripts/birdnet_analysis.py + sed -i "/from subprocess import CalledProcessError/a\import time" "$HOME"/BirdNET-Pi/scripts/birdnet_analysis.py + # Modify main code + sed -i "/os.remove(file.file_name)/i\ processed_size = get_processed_size()" "$HOME"/BirdNET-Pi/scripts/birdnet_analysis.py + sed -i "/os.remove(file.file_name)/i\ if processed_size > 0:" "$HOME"/BirdNET-Pi/scripts/birdnet_analysis.py + sed -i "/os.remove(file.file_name)/i\ move_to_processed(file.file_name, processed_size)" "$HOME"/BirdNET-Pi/scripts/birdnet_analysis.py + sed -i "/os.remove(file.file_name)/i\ else:" "$HOME"/BirdNET-Pi/scripts/birdnet_analysis.py + sed -i "/os.remove(file.file_name)/c\ os.remove(file.file_name)" "$HOME"/BirdNET-Pi/scripts/birdnet_analysis.py +fi || true + diff --git a/battybirdnet-pi/rootfs/etc/cont-init.d/81-modifications.sh b/battybirdnet-pi/rootfs/etc/cont-init.d/81-modifications.sh index 3f72770ab..f6edb33e6 100755 --- a/battybirdnet-pi/rootfs/etc/cont-init.d/81-modifications.sh +++ b/battybirdnet-pi/rootfs/etc/cont-init.d/81-modifications.sh @@ -1,7 +1,18 @@ #!/command/with-contenv bashio -# shellcheck shell=bash +# shellcheck shell=bash disable=SC2016 set -e +################## +# ALLOW RESTARTS # +################## + +if [[ "${BASH_SOURCE[0]}" == /etc/cont-init.d/* ]]; then + mkdir -p /etc/scripts-init + sed -i "s|/etc/cont-init.d|/etc/scripts-init|g" /ha_entrypoint.sh + sed -i "/ rm/d" /ha_entrypoint.sh + cp "${BASH_SOURCE[0]}" /etc/scripts-init/ +fi + ################ # MODIFY WEBUI # ################ @@ -14,74 +25,115 @@ sed -i '/>System Controls/d' "$HOME/BirdNET-Pi/homepage/views.php" # Remove Ram drive option from webui echo "... removing Ram drive from webui as it is handled from HA" -sed -i '/Ram drive/{n;s/center"/center" style="display: none;"/;}' "$HOME/BirdNET-Pi/scripts/service_controls.php" -sed -i '/Ram drive/d' "$HOME/BirdNET-Pi/scripts/service_controls.php" +if grep -q "Ram drive" "$HOME/BirdNET-Pi/scripts/service_controls.php"; then + sed -i '/Ram drive/{n;s/center"/center" style="display: none;"/;}' "$HOME/BirdNET-Pi/scripts/service_controls.php" + sed -i '/Ram drive/d' "$HOME/BirdNET-Pi/scripts/service_controls.php" +fi # Correct services to start as user pi echo "... updating services to start as user pi" -while IFS= read -r file; do - if [[ "$(basename "$file")" != "birdnet_log.service" ]]; then - sed -i "s|ExecStart=|ExecStart=/usr/bin/sudo -u pi |g" "$file" - fi -done < <(find "$HOME/BirdNET-Pi/templates/" -name "b*.service" -print) +if ! grep -q "/usr/bin/sudo" "$HOME/BirdNET-Pi/templates/birdnet_log.service"; then + while IFS= read -r file; do + if [[ "$(basename "$file")" != "birdnet_log.service" ]]; then + sed -i "s|ExecStart=|ExecStart=/usr/bin/sudo -u pi |g" "$file" + fi + done < <(find "$HOME/BirdNET-Pi/templates/" -name "birdnet*.service" -print) +fi # Send services log to container logs echo "... redirecting services logs to container logs" while IFS= read -r file; do - sed -i "/Service/a StandardError=append:/proc/1/fd/1" "$file" - sed -i "/Service/a StandardOutput=append:/proc/1/fd/1" "$file" -done < <(find "$HOME/BirdNET-Pi/templates/" -name "b*.service" -print) + sed -i "/StandardError/d" "$file" + sed -i "/StandardOutput/d" "$file" + sed -i "/\[Service/a StandardError=append:/proc/1/fd/1" "$file" + sed -i "/\[Service/a StandardOutput=append:/proc/1/fd/1" "$file" +done < <(find "$HOME/BirdNET-Pi/templates/" -name "*.service" -print) # Avoid preselection in include and exclude lists echo "... disabling preselecting options in include and exclude lists" sed -i "s|option selected|option disabled|g" "$HOME/BirdNET-Pi/scripts/include_list.php" sed -i "s|option selected|option disabled|g" "$HOME/BirdNET-Pi/scripts/exclude_list.php" +# Preencode API key +if ! grep -q "221160312" "$HOME/BirdNET-Pi/scripts/common.php"; then + sed -i "/return \$_SESSION\['my_config'\];/i\ \ \ \ if (isset(\$_SESSION\['my_config'\]) \&\& empty(\$_SESSION\['my_config'\]\['FLICKR_API_KEY'\])) {\n\ \ \ \ \ \ \ \ \$_SESSION\['my_config'\]\['FLICKR_API_KEY'\] = \"221160312e1c22\";\n\ \ \ \ }" "$HOME"/BirdNET-Pi/scripts/common.php + sed -i "s|e1c22|e1c22ec60ecf336951b0e77|g" "$HOME"/BirdNET-Pi/scripts/common.php +fi + # Correct log services to show /proc/1/fd/1 echo "... redirecting birdnet_log service output to /logs" sed -i "/User=pi/d" "$HOME/BirdNET-Pi/templates/birdnet_log.service" sed -i "s|birdnet_log.sh|cat /proc/1/fd/1|g" "$HOME/BirdNET-Pi/templates/birdnet_log.service" +# Correct backup script +echo "... correct backup script" +sed -i "/PHP_SERVICE=/c PHP_SERVICE=\$(systemctl list-unit-files -t service --no-pager | grep 'php' | grep 'fpm' | awk '{print \$1}')" "$HOME/BirdNET-Pi/scripts/backup_data.sh" + # Caddyfile modifications echo "... modifying Caddyfile configurations" caddy fmt --overwrite /etc/caddy/Caddyfile #Change port to leave 80 free for certificate requests -sed -i "s|http://|http://:8081|g" /etc/caddy/Caddyfile -sed -i "s|http://|http://:8081|g" "$HOME/BirdNET-Pi/scripts/update_caddyfile.sh" -if [ -f /etc/caddy/Caddyfile.original ]; then - rm /etc/caddy/Caddyfile.original +if ! grep -q "http://:8081" /etc/caddy/Caddyfile; then + sed -i "s|http://|http://:8081|g" /etc/caddy/Caddyfile + sed -i "s|http://|http://:8081|g" "$HOME/BirdNET-Pi/scripts/update_caddyfile.sh" + if [ -f /etc/caddy/Caddyfile.original ]; then + rm /etc/caddy/Caddyfile.original + fi fi # Correct webui paths echo "... correcting webui paths" -sed -i "s|/stats|/stats/|g" "$HOME/BirdNET-Pi/homepage/views.php" -sed -i "s|/log|/log/|g" "$HOME/BirdNET-Pi/homepage/views.php" +if ! grep -q "/stats/" "$HOME/BirdNET-Pi/homepage/views.php"; then + sed -i "s|/stats|/stats/|g" "$HOME/BirdNET-Pi/homepage/views.php" + sed -i "s|/log|/log/|g" "$HOME/BirdNET-Pi/homepage/views.php" +fi # Check if port 80 is correctly configured -if [ -n "$(bashio::addon.port 80)" ] && [ "$(bashio::addon.port 80)" != 80 ]; then +if [ -n "$(bashio::addon.port "80")" ] && [ "$(bashio::addon.port "80")" != 80 ]; then bashio::log.fatal "The port 80 is enabled, but should still be 80 if you want automatic SSL certificates generation to work." fi # Correct systemctl path -echo "... updating systemctl path" -mv /helpers/systemctl3.py /bin/systemctl -chmod a+x /bin/systemctl +#echo "... updating systemctl path" +#if [[ -f /helpers/systemctl3.py ]]; then +# mv /helpers/systemctl3.py /bin/systemctl +# chmod a+x /bin/systemctl +#fi + +# Improve streamlit cache +#echo "... add streamlit cache" +#sed -i "/def get_data/i \\@st\.cache_resource\(\)" "$HOME/BirdNET-Pi/scripts/plotly_streamlit.py" + +# Allow reverse proxy for streamlit +echo "... allow reverse proxy for streamlit" +sed -i "s|plotly_streamlit.py --browser.gatherUsageStats|plotly_streamlit.py --server.enableXsrfProtection=false --server.enableCORS=false --browser.gatherUsageStats|g" "$HOME/BirdNET-Pi/templates/birdnet_stats.service" + +# Clean saved mp3 files +echo ".. add highpass and lowpass to sox extracts" +sed -i "s|f'={stop}']|f'={stop}', 'highpass', '250', 'lowpass', '15000']|g" "$HOME/BirdNET-Pi/scripts/utils/reporting.py" +sed -i '/sox.*-V1/s/spectrogram/highpass 250 spectrogram/' "$HOME/BirdNET-Pi/scripts/spectrogram.sh" # Correct timedatectl path echo "updating timedatectl path" -mv /helpers/timedatectl /usr/bin/timedatectl -chown pi:pi /usr/bin/timedatectl -chmod a+x /usr/bin/timedatectl +if [[ -f /helpers/timedatectl ]]; then + mv /helpers/timedatectl /usr/bin/timedatectl + chown pi:pi /usr/bin/timedatectl + chmod a+x /usr/bin/timedatectl +fi # Correct timezone showing in config.php +# shellcheck disable=SC2016 +echo "... updating timezone in config.php" sed -i -e '/