我在phpmailer()中的表单只允许从具有自己域的邮件发送邮件

我有这个代码，这是一个问题，因为当我在表单中指定具有自己域的邮件时，它可以发送邮件。但如果我用gmail或雅虎邮箱，它就不能用了。

这是我在PHPMailer上使用的代码:

<?php
error_reporting(E_ALL);
ini_set('display_errors', '1');
require_once 'phpmailer/PHPMailerAutoload.php';
if (isset($_POST['inputName']) && isset($_POST['inputEmail']) && isset($_POST['inputSubject']) && isset($_POST['inputMessage'])) {
    //check if any of the inputs are empty
    if (empty($_POST['inputName']) || empty($_POST['inputEmail']) || empty($_POST['inputSubject']) || empty($_POST['inputMessage'])) {
        $data = array('success' => false, 'message' => 'Please fill out the form completely.');
        echo json_encode($data);
        exit;
    }
    //create an instance of PHPMailer
    $mail = new PHPMailer();
    $mail->IsSMTP();
    $mail->SMTPAuth = true;
    $mail->Host = "mail.example.com"; // SMTP a utilizar. Por ej. smtp.elserver.com
    $mail->Username = "contacto@example.com"; // Correo completo a utilizar
    $mail->Password = "xxx"; // Contraseña
    $mail->Port = 25; // Puerto a utilizar
    $mail->From = $_POST['inputEmail'];
    $mail->FromName = $_POST['inputName'];
    $mail->AddAddress('contacto@example.net'); //recipient 
    $mail->Subject = $_POST['inputSubject'];
    $mail->Body = "Name: " . $_POST['inputName'] . "'r'n'r'nMessage: " . stripslashes($_POST['inputMessage']);
    $mail->CharSet = 'UTF-8';
    if (isset($_POST['ref'])) {
        $mail->Body .= "'r'n'r'nRef: " . $_POST['ref'];
    }
    if(!$mail->send()) {
        $data = array('success' => false, 'message' => 'El mensaje no se ha enviado: ' . $mail->ErrorInfo);
        echo json_encode($data);
        exit;
    }
    $data = array('success' => true, 'message' => '¡Gracias! Hemos recibido tu mensaje.');
    echo json_encode($data);
} else {
    $data = array('success' => false, 'message' => 'Favor de llenar los campos correctamente.');
    echo json_encode($data);
}