PHP Classes

File: crawl.php

Recommend this page to a friend!
  Classes of Faris AL-Otabi   Searchy PHP Search Engine   crawl.php   Download  
File: crawl.php
Role: Example script
Content type: text/plain
Description: Example script
Class: Searchy PHP Search Engine
Crawl, index and search multiple Web sites
Author: By
Last change:
Date: 1 year ago
Size: 4,420 bytes


Class file image Download

include_once 'vendor/autoload.php';

$httpClient = new \GuzzleHttp\Client();

if (
$url = $_POST['url'];

$response = $httpClient->get($url);

$htmlString = (string) $response->getBody();


$htmlString = mb_convert_encoding($htmlString, 'HTML-ENTITIES', "UTF-8");

$doc = new DOMDocument();


$xpath = new DOMXPath($doc);

$titles = $xpath->evaluate('/html/head/title');
$descriptions = $xpath->evaluate('/html/head/meta[@name="description"]/@content');
$keywords = $xpath->evaluate('/html/head/meta[@name="keywords"]/@content');

$title = '';

$description = '';

$keyword = '';

    foreach (
$titles as $t) {
$title = $t->textContent;

    foreach (
$descriptions as $d) {
$description = $d->textContent;

    foreach (
$keywords as $k) {
$keyword = $k->textContent;

    if (
$title == '') {
$title = $url;

$site = [
'title' => $title,
'blurb' => $description,
'keywords' => $keyword,
'url' => $url

$db = new Database($config);

$index = new SearchEngine($db);

$added = $index->addSite($site);

<!DOCTYPE html>
<html lang="en">

    <meta charset="UTF-8">
    <meta http-equiv="X-UA-Compatible" content="IE=edge">
    <meta name="viewport" content="width=device-width, initial-scale=1.0">
    <title>Searchy - Crawl a Website</title>
    <link href="" rel="stylesheet">
    <link rel="stylesheet" href="" />

    <nav class="navbar navbar-expand-lg navbar-dark bg-dark">
        <div class="container-fluid">
            <a class="navbar-brand" href="#">Searchy</a>
            <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbarColor02" aria-controls="navbarColor02" aria-expanded="false" aria-label="Toggle navigation">
                <span class="navbar-toggler-icon"></span>

            <div class="collapse navbar-collapse" id="navbarColor02">
                <ul class="navbar-nav me-auto">
                    <li class="nav-item">
                        <a class="nav-link" href="index.php">Home</a>
                    <li class="nav-item">
                        <a class="nav-link" href="add.php">Add Site</a>
                    <li class="nav-item">
                        <a class="nav-link active" href="crawl.php">Crawl Site</a>

    <div class="container">
        <?php if (isset($added) && $added == false) : ?>
<div class="d-flex align-items-center justify-content-center mt-3">
                <div class="alert alert-danger">
                    <i class="fa-solid fa-circle-xmark"></i> Site already exist
        <?php elseif (isset($added) && $added == true) : ?>
<div class="d-flex align-items-center justify-content-center mt-3">
                <div class="alert alert-success">
                    <i class="fa-solid fa-circle-check"></i> Site Added
        <?php else : ?>
<?php endif; ?>
<div class="d-flex align-items-center justify-content-center pt-3">
            <div class="card border-dark mb-3" style="max-width: 40rem;">
                <div class="card-header">Crawl a Site</div>
                <div class="card-body">
                    <form class="d-flex" method="POST">
                        <input class="form-control me-sm-2" type="url" name="url" placeholder="URL...">
                        <button class="btn btn-secondary my-2 my-sm-0" type="submit">Submit</button>

    <script src=""></script>
    <script src=""></script>