commit 8fa9179138260884533141842b69c227166a9acd
from: Bradley Taunt <brad@serpapi.com>
date: Thu Feb 15 15:11:45 2024 UTC

Initial testing with crude site size bash script

commit - 3ed6ac65dbd00dda138eb21349938d009e033e03
commit + 8fa9179138260884533141842b69c227166a9acd
blob - bc30a1d1ab8e8e4d39a2b5eb15d701f600022522
blob + 587349164d37aee2848b61df4ad6583b1b030b52
--- _config.yml
+++ _config.yml
@@ -25,6 +25,7 @@ compress_html:
 
 exclude:
   - generate.sh
+  - size.sh
   - README.md
 
 include:
blob - /dev/null
blob + f50689dbf5c18169bd328d7b2a9f0d895b30f968 (mode 755)
--- /dev/null
+++ size.sh
@@ -0,0 +1,54 @@
+#!/bin/bash
+
+if [ $# -eq 0 ]; then
+    echo "Usage: $0 <folder_path>"
+    exit 1
+fi
+
+folder_path=$1
+
+# Timeout value in seconds (adjust as needed)
+url_timeout=10
+
+# Loop through each markdown file in the specified folder
+for file in "$folder_path"/*.md; do
+    if [ -f "$file" ]; then
+        # Extract URL from "pageurl:" YAML
+        url=$(grep "pageurl:" "$file" | awk '{print $2}')
+
+        if [ -n "$url" ]; then
+            download_dir=$(mktemp -d)
+
+            # Use curl with timeout to download the page and its resources, redirecting logs to /dev/null
+            if gtimeout "$url_timeout" curl -L --create-dirs --output "$download_dir/index.html" "$url" > /dev/null 2>&1; then
+                # Display the size of each downloaded resource
+                echo "Resource sizes for $url:"
+                total_size_bytes=0
+
+                # Iterate over files directly in the temporary download directory
+                for resource_file in "$download_dir"/*; do
+                    if [ -f "$resource_file" ]; then
+                        size_bytes=$(stat -f%z "$resource_file")
+                        total_size_bytes=$((total_size_bytes + size_bytes))
+                        echo "$resource_file: $size_bytes bytes"
+                    fi
+                done
+
+                # Convert total size to kilobytes
+                total_size_kb=$((total_size_bytes / 1024))
+
+                # Update the "size:" YAML with the total size
+                awk -v total_size_bytes="$total_size_bytes" '/size:/ {sub(/size:.*/, "size: " total_size_bytes)} 1' "$file" > "$file.tmp" && mv "$file.tmp" "$file"
+
+                # Clean up the temporary download directory
+                rm -r "$download_dir"
+
+                echo "Total size of all resources updated in $file: $total_size_kb kilobytes"
+            else
+                echo "Error: Timeout while checking URL in $file"
+            fi
+        else
+            echo "Error: URL not found in $file"
+        fi
+    fi
+done