Added blocklist updating, main nav tweaks

Implemented Oliphant's unified tier 3 blocklist and the supplementary
instance audit file that tracks how many times an instance has been
blocked by a trusted sources member. Keeping the update function manual
for now to make sure it works smooth, then well automate so it checks on
    it's on at regular intervals.

NOTE: Lists used are located at the following urls:
https://codeberg.org/oliphant/blocklists/src/branch/main/blocklists/_unified_tier3_blocklist.csv
https://codeberg.org/oliphant/blocklists/src/branch/main/blocklists/other/domain_audit_file.csv

Also simplified the main nav to just include a link to the den index
when logged in
about-updates
Ro 1 year ago
parent f9cb8f3a63
commit 7feb76517a
Signed by untrusted user: are0h
GPG Key ID: 29B551CDBD4D3B50

@ -72,8 +72,8 @@ class FrontIndexController extends Controller
public function listings(int $pageNum = 1) public function listings(int $pageNum = 1)
{ {
$range = $pageNum * $this->limit - $this->limit; $range = $pageNum * $this->limit - $this->limit;
$active = Location::where("active", true)->get(); $active = Location::where("active", true)->where('block_count', '>', 2)->get();
$locations = Location::where("active", true) $locations = Location::where("active", true)->where('block_count', '>', 2)
->limit($this->limit)->offset($range)->orderByDesc('id')->get(); ->limit($this->limit)->offset($range)->orderByDesc('id')->get();
$pageCount = ceil(count($active) / $this->limit); $pageCount = ceil(count($active) / $this->limit);

@ -6,10 +6,16 @@ use Illuminate\Http\Request;
use App\Models\Location; use App\Models\Location;
use Ramsey\Uuid\Uuid; use Ramsey\Uuid\Uuid;
use Illuminate\Support\Facades\Auth; use Illuminate\Support\Facades\Auth;
use League\Csv\Reader;
class LocationController extends Controller class LocationController extends Controller
{ {
// //url to oli's unified tier 3 list
private $three = 'https://codeberg.org/oliphant/blocklists/raw/branch/main/blocklists/_unified_tier3_blocklist.csv';
//url to oli's domain audit containin block counts per domain
private $defed = 'https://codeberg.org/oliphant/blocklists/raw/branch/main/blocklists/other/domain_audit_file.csv';
public function addLocation(Request $request) public function addLocation(Request $request)
{ {
$fields = $request->validate([ $fields = $request->validate([
@ -47,4 +53,68 @@ class LocationController extends Controller
]); ]);
} }
} }
public function updateLocations()
{
//$fresh = file($this->three);
//$deny = Reader::createFromPath($fresh, "r");
//$deny->setHeaderOffset(0);
//$list = $deny->getRecords();
//$recordCount = count($fresh);
$duplicates = 0;
$fresh = 0;
$denycount = array_map('str_getcsv', file($this->defed));
$denylist = array_map('str_getcsv', file($this->three));
foreach ($denylist as $item) {
$blockCount = 0;
//get block count for item
foreach ($denycount as $line) {
if ($line[0] == $item[0]) {
$blockcount = $line[1];
}
}
$location = Location::where("url", $item[0])->first();
if ($location) {
++$duplicates;
//update block count for existing item
$location->block_count = $blockcount;
$location->save();
} else {
// make new entries for instances not present
if ($item[0] != 'domain') {
++$fresh;
$new = Location::create([
'uuid' => Uuid::uuid4(),
'name' => $item[0],
'url' => $item[0],
'description' => 'no description',
'active' => true,
'rating' => $item[1],
'added_by' => 1,
'tags' => 'poor moderation, hate speech',
'block_count' => $blockcount
]);
}
}
}
return back()->with('message', $duplicates . ' UPDATED - ' . $fresh . ' CREATED');
//$domain = $csv[1000][0];
//$record = null;
/*
foreach ($blockcount as $line) {
if ($line[0] == $domain) {
$record = $line;
}
}
if ($record != null) {
return back()->with('message', $domain . ' has ' . $record[1] . ' blocks.');
} else {
return back()->with('message', 'NO MATCHES');
}
*/
}
} }

@ -21,5 +21,15 @@ class Location extends Model
protected $fillable = [ protected $fillable = [
"uuid", "uuid",
"name", "name",
"url", "description", "images", "active", "rating", "added_by", "tags", "created_at", "updated_at"]; "url",
"description",
"images",
"active",
"rating",
"added_by",
"tags",
"block_count",
"created_at",
"updated_at"
];
} }

@ -9,7 +9,8 @@
"guzzlehttp/guzzle": "^7.2", "guzzlehttp/guzzle": "^7.2",
"laravel/framework": "^10.10", "laravel/framework": "^10.10",
"laravel/sanctum": "^3.2", "laravel/sanctum": "^3.2",
"laravel/tinker": "^2.8" "laravel/tinker": "^2.8",
"league/csv": "^9.0"
}, },
"require-dev": { "require-dev": {
"fakerphp/faker": "^1.9.1", "fakerphp/faker": "^1.9.1",

90
composer.lock generated

@ -4,7 +4,7 @@
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
"This file is @generated automatically" "This file is @generated automatically"
], ],
"content-hash": "aa322c53454393ed775cfe4807d54a50", "content-hash": "117d0f84e7d090c1b07e9e54e91c9041",
"packages": [ "packages": [
{ {
"name": "brick/math", "name": "brick/math",
@ -1604,6 +1604,94 @@
], ],
"time": "2022-12-11T20:36:23+00:00" "time": "2022-12-11T20:36:23+00:00"
}, },
{
"name": "league/csv",
"version": "9.10.0",
"source": {
"type": "git",
"url": "https://github.com/thephpleague/csv.git",
"reference": "d24b0d484812313b07ab74b0fe4db9661606df6c"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/thephpleague/csv/zipball/d24b0d484812313b07ab74b0fe4db9661606df6c",
"reference": "d24b0d484812313b07ab74b0fe4db9661606df6c",
"shasum": ""
},
"require": {
"ext-json": "*",
"ext-mbstring": "*",
"php": "^8.1.2"
},
"require-dev": {
"doctrine/collections": "^2.1.3",
"ext-dom": "*",
"ext-xdebug": "*",
"friendsofphp/php-cs-fixer": "^v3.22.0",
"phpbench/phpbench": "^1.2.14",
"phpstan/phpstan": "^1.10.26",
"phpstan/phpstan-deprecation-rules": "^1.1.3",
"phpstan/phpstan-phpunit": "^1.3.13",
"phpstan/phpstan-strict-rules": "^1.5.1",
"phpunit/phpunit": "^10.3.1",
"symfony/var-dumper": "^6.3.3"
},
"suggest": {
"ext-dom": "Required to use the XMLConverter and the HTMLConverter classes",
"ext-iconv": "Needed to ease transcoding CSV using iconv stream filters"
},
"type": "library",
"extra": {
"branch-alias": {
"dev-master": "9.x-dev"
}
},
"autoload": {
"files": [
"src/functions_include.php"
],
"psr-4": {
"League\\Csv\\": "src"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Ignace Nyamagana Butera",
"email": "nyamsprod@gmail.com",
"homepage": "https://github.com/nyamsprod/",
"role": "Developer"
}
],
"description": "CSV data manipulation made easy in PHP",
"homepage": "https://csv.thephpleague.com",
"keywords": [
"convert",
"csv",
"export",
"filter",
"import",
"read",
"transform",
"write"
],
"support": {
"docs": "https://csv.thephpleague.com",
"issues": "https://github.com/thephpleague/csv/issues",
"rss": "https://github.com/thephpleague/csv/releases.atom",
"source": "https://github.com/thephpleague/csv"
},
"funding": [
{
"url": "https://github.com/sponsors/nyamsprod",
"type": "github"
}
],
"time": "2023-08-04T15:12:48+00:00"
},
{ {
"name": "league/flysystem", "name": "league/flysystem",
"version": "3.15.1", "version": "3.15.1",

@ -6,6 +6,7 @@
<section> <section>
<article> <article>
<h2>Member Listing </h2> <h2>Member Listing </h2>
<a href="/den/admin/update">UPDATE LOCATIONS</a>
</article> </article>
</section> </section>
@endsection @endsection

@ -7,6 +7,8 @@
<section> <section>
<article> <article>
<h2>Hey {{$handle}} </h2> <h2>Hey {{$handle}} </h2>
<a href="/den/member">Manage Member</a><br />
<a href="/den/locations">Manage Location</a>
</article> </article>
</section> </section>
@endsection @endsection

@ -31,11 +31,8 @@
Listings Listings
</a><br /> </a><br />
@if(Auth::check()) @if(Auth::check())
<a href="/den/member" title="den-member" class="nav-links"> <a href="/den" title="den-start" class="nav-links">
Member Den
</a><br />
<a href="/den/locations" title="den-locations" class="nav-links">
Locations
</a><br /> </a><br />
<a href="/logout" title="logout" class="nav-links"> <a href="/logout" title="logout" class="nav-links">
Logout Logout

@ -9,7 +9,7 @@
{{$pageNum}} of {{$totalPages}} {{$pageNum}} of {{$totalPages}}
<a href="/listings/{{$next}}">NEXT</a><br /> <a href="/listings/{{$next}}">NEXT</a><br />
@foreach($locations as $location) @foreach($locations as $location)
<a role="listitem" href="/location/{{$location->uuid}}">{{$location->name}}</a></a><br /> <a role="listitem" href="/location/{{$location->uuid}}">{{$location->name}} | BLOCK COUNT: {{$location->block_count}}</a><br />
@endforeach @endforeach
<a href="/listings/{{$prev}}">PREV</a> <a href="/listings/{{$prev}}">PREV</a>
{{$pageNum}} of {{$totalPages}} {{$pageNum}} of {{$totalPages}}

@ -35,4 +35,6 @@ Route::group(['prefix' => 'den', 'middleware' => 'member.check'], function () {
Route::get("/member", [DenController::class, 'member']); Route::get("/member", [DenController::class, 'member']);
Route::get("/locations/{action?}", [DenController::class, 'location']); Route::get("/locations/{action?}", [DenController::class, 'location']);
Route::post("/locations/add", [LocationController::class, 'addLocation']); Route::post("/locations/add", [LocationController::class, 'addLocation']);
//admin actions
Route::get("/admin/update", [LocationController::class, 'updateLocations']);
}); });

Loading…
Cancel
Save