Added blocklist updating, main nav tweaks

Implemented Oliphant's unified tier 3 blocklist and the supplementary
instance audit file that tracks how many times an instance has been
blocked by a trusted sources member. Keeping the update function manual
for now to make sure it works smooth, then well automate so it checks on
    it's on at regular intervals.

NOTE: Lists used are located at the following urls:
https://codeberg.org/oliphant/blocklists/src/branch/main/blocklists/_unified_tier3_blocklist.csv
https://codeberg.org/oliphant/blocklists/src/branch/main/blocklists/other/domain_audit_file.csv

Also simplified the main nav to just include a link to the den index
when logged in

app/Http/Controllers/FrontIndexController.php

@@ -72,8 +72,8 @@ class FrontIndexController extends Controller
     public function listings(int $pageNum = 1)
     {
         $range     = $pageNum * $this->limit - $this->limit;
-        $active    = Location::where("active", true)->get();
-        $locations = Location::where("active", true)
+        $active    = Location::where("active", true)->where('block_count', '>', 2)->get();
+        $locations = Location::where("active", true)->where('block_count', '>', 2)
             ->limit($this->limit)->offset($range)->orderByDesc('id')->get();
         $pageCount = ceil(count($active) / $this->limit);
 

app/Http/Controllers/LocationController.php

@@ -6,10 +6,16 @@ use Illuminate\Http\Request;
 use App\Models\Location;
 use Ramsey\Uuid\Uuid;
 use Illuminate\Support\Facades\Auth;
+use League\Csv\Reader;
 
 class LocationController extends Controller
 {
-    //
+    //url to oli's unified tier 3 list
+    private $three = 'https://codeberg.org/oliphant/blocklists/raw/branch/main/blocklists/_unified_tier3_blocklist.csv';
+
+    //url to oli's domain audit containin block counts per domain
+    private $defed = 'https://codeberg.org/oliphant/blocklists/raw/branch/main/blocklists/other/domain_audit_file.csv';
+
     public function addLocation(Request $request)
     {
         $fields = $request->validate([
@@ -47,4 +53,68 @@ class LocationController extends Controller
             ]);
         }
     }
+
+    public function updateLocations()
+    {
+        //$fresh = file($this->three);
+        //$deny  = Reader::createFromPath($fresh, "r");
+        //$deny->setHeaderOffset(0);
+        //$list        = $deny->getRecords();
+        //$recordCount = count($fresh);
+        $duplicates = 0;
+        $fresh      = 0;
+        $denycount  = array_map('str_getcsv', file($this->defed));
+        $denylist   = array_map('str_getcsv', file($this->three));
+
+        foreach ($denylist as $item) {
+            $blockCount = 0;
+            //get block count for item
+            foreach ($denycount as $line) {
+                if ($line[0] == $item[0]) {
+                    $blockcount = $line[1];
+                }
+            }
+            $location = Location::where("url", $item[0])->first();
+            if ($location) {
+                ++$duplicates;
+                //update block count for existing item
+                $location->block_count = $blockcount;
+                $location->save();
+            } else {
+                // make new entries for instances not present
+                if ($item[0] != 'domain') {
+                    ++$fresh;
+                    $new = Location::create([
+                        'uuid'        => Uuid::uuid4(),
+                        'name'        => $item[0],
+                        'url'         => $item[0],
+                        'description' => 'no description',
+                        'active'      => true,
+                        'rating'      => $item[1],
+                        'added_by'    => 1,
+                        'tags'        => 'poor moderation, hate speech',
+                        'block_count' => $blockcount
+                    ]);
+                }
+            }
+        }
+
+        return back()->with('message', $duplicates . ' UPDATED - ' . $fresh . ' CREATED');
+
+        //$domain = $csv[1000][0];
+        //$record = null;
+
+        /*
+        foreach ($blockcount as $line) {
+            if ($line[0] == $domain) {
+                $record = $line;
+            }
+        }
+        if ($record != null) {
+            return back()->with('message', $domain . ' has ' . $record[1] . ' blocks.');
+        } else {
+            return back()->with('message', 'NO MATCHES');
+        }
+        */
+    }
 }

app/Models/Location.php

@@ -21,5 +21,15 @@ class Location extends Model
     protected $fillable   = [
         "uuid",
         "name",
-        "url", "description", "images", "active", "rating", "added_by", "tags", "created_at", "updated_at"];
+        "url",
+        "description",
+        "images",
+        "active",
+        "rating",
+        "added_by",
+        "tags",
+        "block_count",
+        "created_at",
+        "updated_at"
+    ];
 }

composer.json

@@ -9,7 +9,8 @@
         "guzzlehttp/guzzle": "^7.2",
         "laravel/framework": "^10.10",
         "laravel/sanctum": "^3.2",
-        "laravel/tinker": "^2.8"
+        "laravel/tinker": "^2.8",
+        "league/csv": "^9.0"
     },
     "require-dev": {
         "fakerphp/faker": "^1.9.1",

composer.lock

@@ -4,7 +4,7 @@
         "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
         "This file is @generated automatically"
     ],
-    "content-hash": "aa322c53454393ed775cfe4807d54a50",
+    "content-hash": "117d0f84e7d090c1b07e9e54e91c9041",
     "packages": [
         {
             "name": "brick/math",
@@ -1604,6 +1604,94 @@
             ],
             "time": "2022-12-11T20:36:23+00:00"
         },
+        {
+            "name": "league/csv",
+            "version": "9.10.0",
+            "source": {
+                "type": "git",
+                "url": "https://github.com/thephpleague/csv.git",
+                "reference": "d24b0d484812313b07ab74b0fe4db9661606df6c"
+            },
+            "dist": {
+                "type": "zip",
+                "url": "https://api.github.com/repos/thephpleague/csv/zipball/d24b0d484812313b07ab74b0fe4db9661606df6c",
+                "reference": "d24b0d484812313b07ab74b0fe4db9661606df6c",
+                "shasum": ""
+            },
+            "require": {
+                "ext-json": "*",
+                "ext-mbstring": "*",
+                "php": "^8.1.2"
+            },
+            "require-dev": {
+                "doctrine/collections": "^2.1.3",
+                "ext-dom": "*",
+                "ext-xdebug": "*",
+                "friendsofphp/php-cs-fixer": "^v3.22.0",
+                "phpbench/phpbench": "^1.2.14",
+                "phpstan/phpstan": "^1.10.26",
+                "phpstan/phpstan-deprecation-rules": "^1.1.3",
+                "phpstan/phpstan-phpunit": "^1.3.13",
+                "phpstan/phpstan-strict-rules": "^1.5.1",
+                "phpunit/phpunit": "^10.3.1",
+                "symfony/var-dumper": "^6.3.3"
+            },
+            "suggest": {
+                "ext-dom": "Required to use the XMLConverter and the HTMLConverter classes",
+                "ext-iconv": "Needed to ease transcoding CSV using iconv stream filters"
+            },
+            "type": "library",
+            "extra": {
+                "branch-alias": {
+                    "dev-master": "9.x-dev"
+                }
+            },
+            "autoload": {
+                "files": [
+                    "src/functions_include.php"
+                ],
+                "psr-4": {
+                    "League\\Csv\\": "src"
+                }
+            },
+            "notification-url": "https://packagist.org/downloads/",
+            "license": [
+                "MIT"
+            ],
+            "authors": [
+                {
+                    "name": "Ignace Nyamagana Butera",
+                    "email": "nyamsprod@gmail.com",
+                    "homepage": "https://github.com/nyamsprod/",
+                    "role": "Developer"
+                }
+            ],
+            "description": "CSV data manipulation made easy in PHP",
+            "homepage": "https://csv.thephpleague.com",
+            "keywords": [
+                "convert",
+                "csv",
+                "export",
+                "filter",
+                "import",
+                "read",
+                "transform",
+                "write"
+            ],
+            "support": {
+                "docs": "https://csv.thephpleague.com",
+                "issues": "https://github.com/thephpleague/csv/issues",
+                "rss": "https://github.com/thephpleague/csv/releases.atom",
+                "source": "https://github.com/thephpleague/csv"
+            },
+            "funding": [
+                {
+                    "url": "https://github.com/sponsors/nyamsprod",
+                    "type": "github"
+                }
+            ],
+            "time": "2023-08-04T15:12:48+00:00"
+        },
         {
             "name": "league/flysystem",
             "version": "3.15.1",

resources/views/back/member.blade.php

@@ -6,6 +6,7 @@
     <section>
       <article>
         <h2>Member Listing </h2>
+        <a href="/den/admin/update">UPDATE LOCATIONS</a>
       </article>
     </section>
   @endsection

resources/views/back/start.blade.php

@@ -7,6 +7,8 @@
     <section>
       <article>
         <h2>Hey {{$handle}} </h2>
+        <a href="/den/member">Manage Member</a><br />
+        <a href="/den/locations">Manage Location</a>
       </article>
     </section>
   @endsection

resources/views/frame.blade.php

@@ -31,11 +31,8 @@
             Listings
           </a><br />
           @if(Auth::check())
-            <a href="/den/member" title="den-member" class="nav-links">
-              Member
-            </a><br />
-            <a href="/den/locations" title="den-locations" class="nav-links">
-              Locations
+            <a href="/den" title="den-start" class="nav-links">
+              Den
             </a><br />
             <a href="/logout" title="logout" class="nav-links">
               Logout

resources/views/front/listing.blade.php

@@ -9,7 +9,7 @@
         {{$pageNum}} of {{$totalPages}}
         <a href="/listings/{{$next}}">NEXT</a><br />
         @foreach($locations as $location)
-          <a role="listitem" href="/location/{{$location->uuid}}">{{$location->name}}</a></a><br />
+          <a role="listitem" href="/location/{{$location->uuid}}">{{$location->name}} | BLOCK COUNT: {{$location->block_count}}</a><br />
         @endforeach
         <a href="/listings/{{$prev}}">PREV</a>
         {{$pageNum}} of {{$totalPages}}

routes/web.php

@@ -35,4 +35,6 @@ Route::group(['prefix' => 'den', 'middleware' => 'member.check'], function () {
     Route::get("/member", [DenController::class, 'member']);
     Route::get("/locations/{action?}", [DenController::class, 'location']);
     Route::post("/locations/add", [LocationController::class, 'addLocation']);
+    //admin actions
+    Route::get("/admin/update", [LocationController::class, 'updateLocations']);
 });