battybirdnet-pi

battybirdnet-pi/rootfs/helpers/birdnet_to_mqtt.py

@@ -0,0 +1,124 @@
+#! /usr/bin/env python3
+# birdnet_to_mqtt.py
+#
+# Adapted from : https://gist.github.com/deepcoder/c309087c456fc733435b47d83f4113ff
+# Adapted from : https://gist.github.com/JuanMeeske/08b839246a62ff38778f701fc1da5554
+#
+# monitor the records in the syslog file for info from the birdnet system on birds that it detects
+# publish this data to mqtt
+#
+
+import time
+import re
+import dateparser
+import datetime
+import json
+import logging
+import paho.mqtt.client as mqtt
+import subprocess
+
+# Setup basic configuration for logging
+logging.basicConfig(level=logging.INFO)
+
+# this generator function monitors the requested file handle for new lines added at its end
+# the newly added line is returned by the function
+def file_row_generator(s):
+    while True :
+        line = s.readline()
+        if not line:
+            time.sleep(0.1)
+            continue
+        yield line
+
+# mqtt server
+mqtt_server = "%%mqtt_server%%" # server for mqtt
+mqtt_user = "%%mqtt_user%%"  # Replace with your MQTT username
+mqtt_pass = "%%mqtt_pass%%"  # Replace with your MQTT password
+mqtt_port = %%mqtt_port%% # port for mqtt
+
+# mqtt topic for bird heard above threshold will be published
+mqtt_topic_confident_birds = 'birdnet'
+
+# url base for website that will be used to look up info about bird
+bird_lookup_url_base = 'http://en.wikipedia.org/wiki/'
+
+# regular expression patters used to decode the records from birdnet
+re_high_clean = re.compile(r'(?<=^\[birdnet_analysis\]\[INFO\] ).*?(?=\.mp3$)')
+
+syslog = open('/proc/1/fd/1', 'r')
+
+def on_connect(client, userdata, flags, rc, properties=None):
+    """ Callback for when the client receives a CONNACK response from the server. """
+    if rc == 0:
+        logging.info("Connected to MQTT Broker!")
+    else:
+        logging.error(f"Failed to connect, return code {rc}\n")
+
+def get_bird_code(scientific_name):
+    with open('/home/pi/BirdNET-Pi/scripts/ebird.php', 'r') as file:
+        data = file.read()
+
+    # Extract the array from the PHP file
+    array_str = re.search(r'\$ebirds = \[(.*?)\];', data, re.DOTALL).group(1)
+
+    # Convert the PHP array to a Python dictionary
+    bird_dict = {re.search(r'"(.*?)"', line).group(1): re.search(r'=> "(.*?)"', line).group(1)
+                 for line in array_str.split('\n') if '=>' in line}
+
+    # Return the corresponding value for the given bird's scientific name
+    return bird_dict.get(scientific_name)
+
+# this little hack is to make each received record for the all birds section unique
+# the date and time that the log returns is only down to the 1 second accuracy, do
+# you can get multiple records with same date and time, this will make Home Assistant not
+# think there is a new reading so we add a incrementing tenth of second to each record received
+ts_noise = 0.0
+
+#try :
+# connect to MQTT server
+mqttc = mqtt.Client('birdnet_mqtt')  # Create instance of client with client ID
+mqttc.username_pw_set(mqtt_user, mqtt_pass) # Use credentials
+mqttc.connect(mqtt_server, mqtt_port)  # Connect to (broker, port, keepalive-time)
+mqttc.on_connect = on_connect
+mqttc.loop_start()
+
+# call the generator function and process each line that is returned
+for row in file_row_generator(syslog):
+    # bird found above confidence level found, process it
+    if re_high_clean.search(row) :
+
+        # this slacker regular expression work, extracts the data about the bird found from the log line
+        # I do the parse in two passes, because I did not know the re to do it in one!
+
+        raw_high_bird = re.search(re_high_clean, row)
+        raw_high_bird = raw_high_bird.group(0)
+
+        # the fields we want are separated by semicolons, so split
+        high_bird_fields = raw_high_bird.split(';')
+
+        # build a structure in python that will be converted to json
+        bird = {}
+
+        # human time in this record is in two fields, date and time. They are human format
+        # combine them together separated by a space and they turn the human data into a python
+        # timestamp
+        raw_ts = high_bird_fields[0] + ' ' + high_bird_fields[1]
+
+        #bird['ts'] = str(datetime.datetime.timestamp(dateparser.parse(raw_ts)))
+        bird['Date'] = high_bird_fields[0]
+        bird['Time'] = high_bird_fields[1]
+        bird['ScientificName'] = high_bird_fields[2]
+        bird['CommonName'] = high_bird_fields[3]
+        bird['Confidence'] = high_bird_fields[4]
+        bird['SpeciesCode'] = get_bird_code(high_bird_fields[2])
+        bird['ClipName'] = high_bird_fields[11]
+
+        # build a url from scientific name of bird that can be used to lookup info about bird
+        bird['url'] = bird_lookup_url_base + high_bird_fields[2].replace(' ', '_')
+
+        # convert to json string we can sent to mqtt
+        json_bird = json.dumps(bird)
+
+        print('Posted to MQTT : ok')
+
+        mqttc.publish(mqtt_topic_confident_birds, json_bird, 1)

battybirdnet-pi/rootfs/helpers/birdnet_to_mqtt.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/with-contenv bashio
+# shellcheck shell=bash
+
+echo "Starting service: mqtt automated publish"
+"$PYTHON_VIRTUAL_ENV" /usr/bin/birdnet_to_mqtt.py &>/proc/1/fd/1

battybirdnet-pi/rootfs/helpers/caddy_ingress.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+# shellcheck shell=bash
+
+# Get values
+source /etc/birdnet/birdnet.conf
+
+# Create ingress configuration for Caddyfile
+  cat << EOF >> /etc/caddy/Caddyfile
+:8082 {
+  root * ${EXTRACTED}
+  file_server browse
+  handle /By_Date/* {
+    file_server browse
+  }
+  handle /Charts/* {
+    file_server browse
+  }
+  reverse_proxy /stream localhost:8000
+  php_fastcgi unix//run/php/php-fpm.sock
+  reverse_proxy /log* localhost:8080
+  reverse_proxy /stats* localhost:8501
+  reverse_proxy /terminal* localhost:8888
+}
+EOF

battybirdnet-pi/rootfs/helpers/convert_list.php

@@ -0,0 +1,116 @@
+<meta name="viewport" content="width=device-width, initial-scale=1">
+<style>
+</style>
+
+<p><strong>This tool will allow to convert on-the-fly species to compensate for model errors. It SHOULD NOT BE USED except if you know what you are doing, instead the model errors should be reported to the owner. However, it is still convenient for systematic biases that are confirmed through careful listening of samples, while waiting for the models to be updated.</strong></p>
+
+<div class="customlabels column1">
+<form action="" method="GET" id="add">
+  <input type="hidden" id="species" name="species">
+  <h3>Specie to convert from :</h3>
+  <!-- Input box to filter options in the first table -->
+  <input type="text" id="species1Search" onkeyup="filterOptions('species1')" placeholder="Search for species...">
+  <select name="species1" id="species1" size="25">
+  <?php
+    error_reporting(E_ALL);
+    ini_set('display_errors',1);
+
+    $filename = './scripts/labels.txt';
+    $eachline = file($filename, FILE_IGNORE_NEW_LINES);
+
+    foreach($eachline as $lines){echo
+  "<option value=\"".$lines."\">$lines</option>";}
+  ?>
+  </select>
+  <br><br> <!-- Added a space between the two tables -->
+  <h3>Specie to convert to :</h3>
+  <!-- Input box to filter options in the second table -->
+  <input type="text" id="species2Search" onkeyup="filterOptions('species2')" placeholder="Search for species...">
+  <select name="species2" id="species2" size="25">
+  <?php
+    foreach($eachline as $lines){echo
+  "<option value=\"".$lines."\">$lines</option>";}
+  ?>
+  </select>
+  <input type="hidden" name="add" value="add">
+</form>
+<div class="customlabels smaller">
+  <button type="submit" name="view" value="Converted" form="add">>>ADD>></button>
+</div>
+</div>
+
+<div class="customlabels column2">
+  <table><td>
+  <button type="submit" name="view" value="Converted" form="add">>>ADD>></button>
+  <br><br>
+  <button type="submit" name="view" value="Converted" form="del">REMOVE</button>
+  </td></table>
+</div>
+
+<div class="customlabels column3" style="margin-top: 0;"> <!-- Removed the blank space above the table -->
+<form action="" method="GET" id="del">
+  <h3>Converted Species List</h3>
+  <select name="species[]" id="value2" multiple size="25">
+<?php
+  $filename = './scripts/convert_species_list.txt'; // Changed the file path
+  $eachline = file($filename, FILE_IGNORE_NEW_LINES);
+  foreach($eachline as $lines){
+    echo
+  "<option value=\"".$lines."\">$lines</option>";
+}?>
+  </select>
+  <input type="hidden" name="del" value="del">
+</form>
+<div class="customlabels smaller">
+  <button type="submit" name="view" value="Converted" form="del">REMOVE</button>
+</div>
+</div>
+
+<input type="hidden" id="hiddenSpecies" name="hiddenSpecies">
+
+<script>
+    document.getElementById("add").addEventListener("submit", function(event) {
+      var speciesSelect1 = document.getElementById("species1");
+      var speciesSelect2 = document.getElementById("species2");
+      if (speciesSelect1.selectedIndex < 0 || speciesSelect2.selectedIndex < 0) {
+        alert("Please select a species from both lists.");
+        document.querySelector('.views').style.opacity = 1;
+        event.preventDefault();
+      } else {
+        var selectedSpecies1 = speciesSelect1.options[speciesSelect1.selectedIndex].value;
+        var selectedSpecies2 = speciesSelect2.options[speciesSelect2.selectedIndex].value;
+        document.getElementById("species").value = selectedSpecies1 + ";" + selectedSpecies2;
+      }
+    });
+
+    // Store the original list of options in a variable
+    var originalOptions = {};
+    
+    // Function to filter options in a select element
+    function filterOptions(id) {
+      var input = document.getElementById(id + "Search");
+      var filter = input.value.toUpperCase();
+      var select = document.getElementById(id);
+      var options = select.getElementsByTagName("option");
+    
+      // If the original list of options for this select element hasn't been stored yet, store it
+      if (!originalOptions[id]) {
+        originalOptions[id] = Array.from(options).map(option => option.value);
+      }
+    
+      // Clear the select element
+      while (select.firstChild) {
+        select.removeChild(select.firstChild);
+      }
+    
+      // Populate the select element with the filtered labels
+      originalOptions[id].forEach(label => {
+        if (label.toUpperCase().indexOf(filter) > -1) {
+          let option = document.createElement('option');
+          option.value = label;
+          option.text = label;
+          select.appendChild(option);
+        }
+      });
+    }
+</script>

battybirdnet-pi/rootfs/helpers/spectral_analysis.py

@@ -0,0 +1,70 @@
+import numpy as np
+import scipy.io.wavfile as wavfile
+import matplotlib.pyplot as plt
+import os
+import glob
+import sys  # Import the sys module
+
+from utils.helpers import get_settings
+
+# Dependencies /usr/bin/pip install numpy scipy matplotlib
+
+# Define the directory containing the WAV files
+conf = get_settings()
+input_directory = os.path.join(conf['RECS_DIR'], 'StreamData')
+output_directory = os.path.join(conf['RECS_DIR'], 'Extracted/Charts')
+
+# Ensure the output directory exists
+if not os.path.exists(output_directory):
+    os.makedirs(output_directory)
+
+# Check if a command-line argument is provided
+if len(sys.argv) > 1:
+    # If an argument is provided, use it as the file to analyze
+    wav_files = [sys.argv[1]]
+else:
+    # If no argument is provided, analyze all WAV files in the directory
+    wav_files = glob.glob(os.path.join(input_directory, '*.wav'))
+
+# Process each file
+for file_path in wav_files:
+    # Load the WAV file
+    sample_rate, audio_data = wavfile.read(file_path)
+
+    # If stereo, select only one channel
+    if len(audio_data.shape) > 1:
+        audio_data = audio_data[:, 0]
+
+    # Apply the Hamming window to the audio data
+    hamming_window = np.hamming(len(audio_data))
+    windowed_data = audio_data * hamming_window
+
+    # Compute the FFT of the windowed audio data
+    audio_fft = np.fft.fft(windowed_data)
+    audio_fft = np.abs(audio_fft)
+
+    # Compute the frequencies associated with the FFT values
+    frequencies = np.fft.fftfreq(len(windowed_data), d=1/sample_rate)
+
+    # Select the range of interest
+    idx = np.where((frequencies >= 150) & (frequencies <= 15000))
+
+    # Calculate the saturation threshold based on the bit depth
+    bit_depth = audio_data.dtype.itemsize * 8
+    max_amplitude = 2**(bit_depth - 1) - 1
+    saturation_threshold = 0.8 * max_amplitude
+
+    # Plot the spectrum with a logarithmic Y-axis
+    plt.figure(figsize=(10, 6))
+    plt.semilogy(frequencies[idx], audio_fft[idx], label='Spectrum')
+    plt.axhline(y=saturation_threshold, color='r', linestyle='--', label='Saturation Threshold')
+    plt.xlabel("Frequency (Hz)")
+    plt.ylabel("Amplitude (Logarithmic)")
+    plt.title(f"Frequency Spectrum (150 - 15000 Hz) - {os.path.basename(file_path)}")
+    plt.legend()
+    plt.grid(True)
+
+    # Save the plot as a PNG file
+    output_filename = os.path.basename(file_path).replace('.wav', '_spectrum.png')
+    plt.savefig(os.path.join(output_directory, output_filename))
+    plt.close()  # Close the figure to free memory

battybirdnet-pi/rootfs/helpers/spectral_analysis.sh

@@ -0,0 +1,62 @@
+#!/usr/bin/env bash
+# Performs the recording from the specified RTSP stream or soundcard
+source /etc/birdnet/birdnet.conf
+
+# Read the logging level from the configuration option
+LOGGING_LEVEL="${LogLevel_BirdnetRecordingService}"
+# If empty for some reason default to log level of error
+[ -z "$LOGGING_LEVEL" ] && LOGGING_LEVEL='error'
+# Additionally if we're at debug or info level then allow printing of script commands and variables
+if [ "$LOGGING_LEVEL" == "info" ] || [ "$LOGGING_LEVEL" == "debug" ];then
+  # Enable printing of commands/variables etc to terminal for debugging
+  set -x
+fi
+
+[ -z "$RECORDING_LENGTH" ] && RECORDING_LENGTH=15
+[ -d "$RECS_DIR"/StreamData ] || mkdir -p "$RECS_DIR"/StreamData
+
+filename="Spectrum_$(date "+%Y-%m-%d_%H:%M").wav"
+
+if [ ! -z "$RTSP_STREAM" ];then
+  # Explode the RSPT steam setting into an array so we can count the number we have
+  RTSP_STREAMS_EXPLODED_ARRAY=("${RTSP_STREAM//,/ }")
+
+  while true;do
+
+    # Initially start the count off at 1 - our very first stream
+    RTSP_STREAMS_STARTED_COUNT=1
+    FFMPEG_PARAMS=""
+
+    # Loop over the streams
+    for i in "${RTSP_STREAMS_EXPLODED_ARRAY[@]}"
+    do
+      # Map id used to map input to output (first stream being 0), this is 0 based in ffmpeg so decrement our counter (which is more human readable) by 1
+      MAP_ID="$((RTSP_STREAMS_STARTED_COUNT-1))"
+      # Build up the parameters to process the RSTP stream, including mapping for the output
+      FFMPEG_PARAMS+="-vn -thread_queue_size 512 -i ${i} -map ${MAP_ID}:a:0 -t ${RECORDING_LENGTH} -acodec pcm_s16le -ac 2 -ar 48000 file:${RECS_DIR}/StreamData/$filename "
+      # Increment counter
+      ((RTSP_STREAMS_STARTED_COUNT += 1))
+    done
+
+  # Make sure were passing something valid to ffmpeg, ffmpeg will run interactive and control our loop by waiting ${RECORDING_LENGTH} between loops because it will stop once that much has been recorded
+  if [ -n "$FFMPEG_PARAMS" ];then
+    ffmpeg -hide_banner -loglevel "$LOGGING_LEVEL" -nostdin "$FFMPEG_PARAMS"
+  fi
+
+  done
+else
+  if pgrep arecord &> /dev/null ;then
+    echo "Recording"
+  else
+    if [ -z "${REC_CARD}" ];then
+      arecord -f S16_LE -c"${CHANNELS}" -r48000 -t wav --max-file-time "${RECORDING_LENGTH}"\
+	      	      	       --use-strftime "${RECS_DIR}"/StreamData/"$filename"
+    else
+      arecord -f S16_LE -c"${CHANNELS}" -r48000 -t wav --max-file-time "${RECORDING_LENGTH}"\
+        -D "${REC_CARD}" --use-strftime "${RECS_DIR}"/StreamData/"$filename"
+    fi
+  fi
+fi
+
+# Create the spectral analysis
+"$PYTHON_VIRTUAL_ENV" "$HOME"/BirdNET-Pi/scripts/spectral_analysis.py

battybirdnet-pi/rootfs/helpers/timedatectl

@@ -0,0 +1,72 @@
+#!/bin/bash
+
+# Function to show the current timezone, with two alternative methods
+show_timezone() {
+    # Check if the /etc/timezone file exists
+    if [ -f /etc/timezone ]; then
+        timezone=$(cat /etc/timezone)
+    elif [ -f /etc/localtime ]; then
+        timezone=$(readlink /etc/localtime)
+        timezone=${timezone/\/usr\/share\/zoneinfo\//}
+    else
+        timezone="Cannot determine timezone."
+    fi
+    echo "$timezone"
+}
+
+# Function to set the timezone
+set_timezone() {
+    new_timezone="$1"
+    echo "$new_timezone" | sudo tee /etc/timezone >/dev/null
+    sudo ln -sf /usr/share/zoneinfo/"$new_timezone" /etc/localtime
+    if [ -f /etc/environment ]; then sudo sed -i "/TZ/c\TZ=$new_timezone" /etc/environment; fi
+    if [ -d /var/run/s6/container_environment ]; then echo "$new_timezone" | sudo tee /var/run/s6/container_environment/TZ > /dev/null; fi
+    echo "$new_timezone"
+}
+
+# Main script
+case "$1" in
+    "set-ntp")
+        case "$2" in
+            "false")
+                sudo systemctl stop systemd-timesyncd
+                sudo systemctl disable systemd-timesyncd
+                echo "NTP disabled"
+                ;;
+            "true")
+                sudo systemctl start systemd-timesyncd
+                sudo systemctl enable systemd-timesyncd
+                echo "NTP enabled"
+                ;;
+            *)
+                echo "Invalid argument for set-ntp. Use 'false' or 'true'."
+                ;;
+        esac
+        ;;
+    "show")
+        show_timezone
+        ;;
+    "set-timezone")
+        set_timezone "$2"
+        ;;
+    *)
+        # Get values
+        local_time="$(date)"
+        utc_time="$(date -u)"
+        time_zone="$(show_timezone)"
+        # Check if NTP is used
+        if sudo systemctl status systemd-timesyncd | grep -q " active"; then
+            ntp_status="yes"
+            ntp_service="active"
+        else
+            ntp_status="no"
+            ntp_service="inactive"
+        fi
+        # Print the information
+        echo "Local time: $local_time"
+        echo "Universal time: $utc_time"
+        echo "Time zone: $time_zone"
+        echo "Network time on: $ntp_status"
+        echo "NTP service: $ntp_service"
+        ;;
+esac

battybirdnet-pi/rootfs/helpers/views.add

@@ -0,0 +1,27 @@
+  if($_GET['view'] == "Converted"){
+    ensure_authenticated();
+	if(isset($_GET['species']) && isset($_GET['add'])){
+	  $file = './scripts/convert_species_list.txt';
+	  $str = file_get_contents("$file");
+	  $str = preg_replace("/(^[\r\n]*|[\r\n]+)[\s\t]*[\r\n]+/", "\n", $str);
+	  file_put_contents("$file", "$str");
+	  // Write $_GET['species'] to the file
+	  file_put_contents("./scripts/convert_species_list.txt", htmlspecialchars_decode($_GET['species'], ENT_QUOTES)."\n", FILE_APPEND);
+      } elseif (isset($_GET['species']) && isset($_GET['del'])){
+      $file = './scripts/convert_species_list.txt';
+      $str = file_get_contents("$file");
+      $str = preg_replace('/^\h*\v+/m', '', $str);
+      file_put_contents("$file", "$str");
+      foreach($_GET['species'] as $selectedOption) {
+        $content = file_get_contents("./scripts/convert_species_list.txt");
+        $newcontent = str_replace($selectedOption, "", "$content");
+        $newcontent = str_replace(htmlspecialchars_decode($selectedOption, ENT_QUOTES), "", "$content");
+        file_put_contents("./scripts/convert_species_list.txt", "$newcontent");
+      }
+      $file = './scripts/convert_species_list.txt';
+      $str = file_get_contents("$file");
+      $str = preg_replace('/^\h*\v+/m', '', $str);
+      file_put_contents("$file", "$str");
+    }
+    include('./scripts/convert_list.php');
+  }